Prepare clean Sargo images and validated storage inventories

Keep internal documents out of system images while retaining license notices and package versions. Generate boot mappings from a bounded storage inventory, and allow long SSH diagnostics. Hardware boot qualification remains pending.

tundra/tests/test_storage_layout.py

@@ -0,0 +1,37 @@
+# SPDX-License-Identifier: MPL-2.0
+from pathlib import Path
+import sys
+import unittest
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]/'tools'))
+from storage_layout import mount_table
+
+
+class StorageLayoutTests(unittest.TestCase):
+    def layout(self):
+        return dict(schema=2, device='/dev/mmcblk0p72', partitionLabel='userdata',
+                    filesystem='ext4', backingDevice='259:40', size=53648801280,
+                    table='0 104448000 linear 259:40 329728', firstMiBSha256='a'*64)
+
+    def test_supported_inventory_uses_partition_path(self):
+        self.assertEqual(mount_table(self.layout()),
+                         '0 104448000 linear /dev/mmcblk0p72 329728')
+
+    def test_different_inventory_offset_is_not_replaced_with_developer_offset(self):
+        layout=self.layout(); layout['table']='0 104000000 linear 259:40 8192'
+        self.assertEqual(mount_table(layout), '0 104000000 linear /dev/mmcblk0p72 8192')
+
+    def test_rejects_overflow_other_partition_and_multiple_extents(self):
+        for patch in ({'table':'0 104448000 linear 259:40 999999999'},
+                      {'device':'/dev/mmcblk0p1'}, {'filesystem':'crypto_LUKS'},
+                      {'backingDevice':'259:41'},
+                      {'table':'0 104448000 linear 259:40 329728\n1 2 linear 259:40 3'},
+                      {'firstMiBSha256':'not-a-hash'}):
+            with self.subTest(patch=patch), self.assertRaises(ValueError):
+                mount_table(self.layout() | patch)
+
+    def test_legacy_inventory_remains_supported_but_not_generalized(self):
+        layout=self.layout();layout.pop('schema')
+        self.assertIn('329728',mount_table(layout))
+        layout['table']='0 104000000 linear 259:40 8192'
+        with self.assertRaises(ValueError):mount_table(layout)

tundra/tests/test_system_image.py

@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: MPL-2.0
 """Exercise ownership/format safeguards without writing to a real device."""
 import importlib.util
+import json
 import os
 from pathlib import Path
 import tempfile
@@ -14,6 +15,32 @@ spec.loader.exec_module(image)
 
 
 class SystemImageTests(unittest.TestCase):
+    def test_distribution_omits_working_documents_but_keeps_component_licenses(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            root=Path(temporary); product=root/'product'; reference=root/'reference'
+            (product/'component').mkdir(parents=True);reference.mkdir()
+            (product/'README.md').write_text('Internal working document')
+            (product/'component/LICENSE.md').write_text('Original upstream notice')
+            (reference/'manifest.json').write_text(json.dumps({'packages':{
+                'example':{'binary:Package':'example','Version':'1',
+                           'source:Package':'example','source:Version':'1',
+                           'localPath':'private/path'}}}))
+            image.prepare_distribution_metadata(product,reference)
+            self.assertFalse((product/'README.md').exists())
+            self.assertEqual((product/'component/LICENSE').read_text(),'Original upstream notice')
+            inventory=json.loads((product/'runtime-packages.json').read_text())
+            self.assertEqual(inventory['packages'][0]['binary:Package'],'example')
+            self.assertNotIn('localPath',inventory['packages'][0])
+            self.assertTrue((product/'NOTICE').is_file())
+
+    def test_conflicting_project_license_is_not_overwritten(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            product=Path(temporary)
+            (product/'LICENSE').write_text('Keep this upstream notice')
+            with self.assertRaisesRegex(ValueError,'existing notice'):
+                image.prepare_distribution_metadata(product,Path('/unused'))
+            self.assertEqual((product/'LICENSE').read_text(),'Keep this upstream notice')
+
     def test_pack_refuses_real_ownership_changes_outside_fakeroot(self):
         with patch.dict(os.environ, {}, clear=True), patch.object(image.os, 'lchown') as chown:
             with self.assertRaisesRegex(RuntimeError, 'fakeroot'):

tundra/tools/build-rescue-boot.py

@@ -16,6 +16,7 @@ import time
 from cryptography.hazmat.primitives.asymmetric import rsa, ec
 from cryptography.hazmat.primitives import serialization
 from ramdisk_reference import read_newc, select_helpers
+from storage_layout import mount_table
 from tundra import ROOT, cpio, checked_source, digest, save, linker
 
 
@@ -67,8 +68,7 @@ def main():
     out=args.output.resolve();out.mkdir(parents=True,exist_ok=False)
     plan,_=load_boot().checked_plan()
     layout=json.loads(args.layout.read_text())
-    if layout['device']!='/dev/mmcblk0p72' or layout['table']!='0 104448000 linear 259:40 329728' or int(layout['size'])!=53648801280:
-        raise ValueError('Recipe only supports the inventoried read-only Sargo layout')
+    storage_table = mount_table(layout)
     source=Path(plan['backupDirectory'])/'boot_a.img'
     packager=checked_source('aosp-mkbootimg')
     unpack=out/'original'
@@ -102,7 +102,10 @@ def main():
     (out/'known_hosts').write_text('[10.15.19.82]:2222 '+hostpublic+'\n[127.0.0.1]:2222 '+hostpublic+'\n')
     init=(ROOT/'boot/rescue-init.sh').read_bytes().replace(b'# No LVM activation',
         ('EXPECTED_STORAGE_HASH='+shlex.quote(layout['firstMiBSha256'])+'\n# No LVM activation').encode())
-    sources=[Path(__file__),ROOT/'boot/rescue-init.sh',ROOT/'tools/ramdisk_reference.py']
+    init=init.replace(b"'0 104448000 linear /dev/mmcblk0p72 329728'",
+                      shlex.quote(storage_table).encode())
+    sources=[Path(__file__),ROOT/'boot/rescue-init.sh',ROOT/'tools/ramdisk_reference.py',
+             ROOT/'tools/storage_layout.py']
     if state:
         # Keep the existing layout intact; only /tundra/data/<UUID>.ext4 is used
         # for Tundra state. Opening its containing filesystem RW is necessary.

tundra/tools/build-system-image.py

@@ -21,6 +21,39 @@ def digest(path):
         return hashlib.file_digest(stream, 'sha256').hexdigest()
 
 
+def prepare_distribution_metadata(product, reference):
+    """Keep license notices, but leave working documents out of the image."""
+    removed = []
+    for path in sorted(product.rglob('*')):
+        if not path.is_file() or path.is_symlink():
+            continue
+        if path.suffix.lower() != '.md':
+            continue
+        if path.stem.lower() in ('license', 'copying', 'notice'):
+            target = path.with_suffix('')
+            if target.exists() and target.read_bytes() != path.read_bytes():
+                raise ValueError('Conflicting license notices: '+str(path))
+            if not target.exists():
+                path.rename(target)
+            else:
+                path.unlink()
+        else:
+            removed.append(str(path.relative_to(product)))
+            path.unlink()
+    for name in ('LICENSE', 'NOTICE'):
+        if (product/name).exists() and (product/name).read_bytes() != (PROJECT/name).read_bytes():
+            raise ValueError('Refusing to replace an existing notice: '+name)
+        shutil.copy2(PROJECT/name, product/name)
+    manifest = json.loads((reference/'manifest.json').read_text())
+    packages = [{key: info[key] for key in
+                 ('binary:Package', 'Version', 'Architecture', 'source:Package', 'source:Version')
+                 if key in info} for info in manifest['packages'].values()]
+    (product/'runtime-packages.json').write_text(
+        json.dumps({'schema': 1, 'packages': packages}, indent=2)+'\n')
+    return {'internalMarkdownRemoved': removed, 'packageCount': len(packages),
+            'licenseNoticesRetained': True}
+
+
 def module(name, path):
     spec = importlib.util.spec_from_file_location(name, path)
     result = importlib.util.module_from_spec(spec)
@@ -84,6 +117,7 @@ def main():
     shutil.copytree(ROOT/'droidian', product/'trial', ignore=shutil.ignore_patterns('__pycache__'))
     for name in ('libEGL_vulpes_hybris.so', 'manifest.json'):
         shutil.copy2(args.product/'trial/graphics'/name, product/'trial/graphics'/name)
+    distribution = prepare_distribution_metadata(product, args.reference)
     for name in ('home/droidian', 'etc/dbus-1', 'reference'):
         (root/name).mkdir(parents=True, exist_ok=True)
     shutil.copy2(android, root/'reference/android-rootfs.img')
@@ -116,6 +150,7 @@ RemainAfterExit=yes
                 'androidSha256': ANDROID_SHA, 'engine': lock,
                 'release': json.loads((product/'release.json').read_text()),
                 'staging': staging, 'filesystemCheckPassed': True,
+                'distributionContents': distribution,
                 'flashReady': False, 'hardwareBootTested': False,
                 'scope': 'Self-contained userspace; vendor/firmware partitions still required; RAM boot candidate',
                 'sourceHashes': {str(p.relative_to(PROJECT)): digest(p) for p in

tundra/tools/inventory-storage.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: MPL-2.0
+"""Inventory an existing Sargo ext4/LVM mount without changing any storage."""
+import argparse
+import hashlib
+import json
+import os
+from pathlib import Path
+import stat
+import subprocess
+
+from storage_layout import mount_table
+
+
+def command(*args):
+    return subprocess.check_output(args, text=True, timeout=15,
+        env={**os.environ, 'PATH': '/usr/sbin:/usr/bin:/sbin:/bin', 'LC_ALL': 'C'}).strip()
+
+
+def inventory(mountpoint):
+    device = Path('/dev/mmcblk0p72')
+    if not stat.S_ISBLK(device.stat().st_mode):
+        raise ValueError('Sargo userdata block device is missing')
+    label = command('blkid', '-s', 'PARTLABEL', '-o', 'value', str(device))
+    if label != 'userdata':
+        raise ValueError('Unexpected partition label')
+    mounted = json.loads(command('findmnt', '--json', '--mountpoint', mountpoint,
+                                 '--output', 'SOURCE,FSTYPE'))['filesystems']
+    if len(mounted) != 1 or mounted[0]['fstype'] != 'ext4':
+        raise ValueError('Expected one existing ext4 mount')
+    source = mounted[0]['source']
+    if not source.startswith('/dev/mapper/'):
+        raise ValueError('Only the existing single-extent LVM layout is supported')
+    table = command('dmsetup', 'table', source)
+    with device.open('rb', buffering=0) as stream:
+        first = stream.read(1048576)
+    if len(first) != 1048576:
+        raise ValueError('Cannot read userdata identity')
+    layout = {
+        'schema': 2, 'device': str(device), 'partitionLabel': label,
+        'filesystem': mounted[0]['fstype'], 'table': table,
+        'backingDevice': Path('/sys/class/block/mmcblk0p72/dev').read_text().strip(),
+        'size': int(command('blockdev', '--getsize64', str(device))),
+        'firstMiBSha256': hashlib.sha256(first).hexdigest(),
+    }
+    mount_table(layout)
+    return layout
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--mountpoint', required=True,
+                        help='Existing root mount, or /reference/installed under Tundra')
+    args = parser.parse_args()
+    print(json.dumps(inventory(args.mountpoint), indent=2))

tundra/tools/rescue-ssh.py

@@ -24,7 +24,7 @@ def main():
     if not (folder/'boot-report.json').is_file():
         raise ValueError('This image has not been sent with the checked boot tool')
     ssh=['ssh','-F','/dev/null','-o','BatchMode=yes','-o','IdentitiesOnly=yes',
-         '-o','ConnectTimeout=5','-o','ServerAliveInterval=5','-o','ServerAliveCountMax=6',
+         '-o','ConnectTimeout=5','-o','ServerAliveInterval=5','-o','ServerAliveCountMax='+('60' if args.stream else '6'),
          '-o','StrictHostKeyChecking=yes','-o','HostKeyAlgorithms=+ssh-rsa',
          '-o','UserKnownHostsFile='+str(folder/'known_hosts'),'-i',str(folder/'client-key'),
          '-p','2222','root@10.15.19.82']

tundra/tools/storage_layout.py

@@ -0,0 +1,32 @@
+# SPDX-License-Identifier: MPL-2.0
+"""Validate an inventoried, single-extent Sargo userdata mapping."""
+import re
+
+
+def mount_table(layout):
+    device = layout.get('device')
+    if device != '/dev/mmcblk0p72':
+        raise ValueError('Expected Sargo userdata, not another partition')
+    if not re.fullmatch(r'[0-9a-f]{64}', layout.get('firstMiBSha256', '')):
+        raise ValueError('Missing userdata identity checksum')
+    match = re.fullmatch(r'0 ([0-9]+) linear ([0-9]+:[0-9]+) ([0-9]+)',
+                         layout.get('table', ''))
+    if not match:
+        raise ValueError('Only a single linear extent is supported')
+    sectors, backing, offset = int(match[1]), match[2], int(match[3])
+    size = int(layout['size'])
+    if size <= 0 or size % 512 or sectors <= 0 or offset < 2048:
+        raise ValueError('Invalid userdata extent geometry')
+    if (sectors + offset) * 512 > size:
+        raise ValueError('Extent exceeds userdata')
+    if layout.get('schema') == 2:
+        if (layout.get('partitionLabel') != 'userdata' or
+                layout.get('filesystem') != 'ext4' or
+                layout.get('backingDevice') != backing):
+            raise ValueError('Mapping must be ext4 on the inventoried userdata device')
+    elif not (layout['table'] == '0 104448000 linear 259:40 329728'
+              and size == 53648801280):
+        raise ValueError('A new layout requires a schema 2 inventory')
+    # Kernel major/minor numbers can change between boots. The partition path
+    # is stable with this Sargo kernel, and its first MiB is checked before use.
+    return f'0 {sectors} linear {device} {offset}'