From 9b606ed5f35973044d2345e49631e70a3e0da804 Mon Sep 17 00:00:00 2001 From: OverSu Date: Thu, 1 Oct 2026 12:25:53 +0200 Subject: [PATCH] Prepare clean Sargo images and validated storage inventories Keep internal documents out of system images while retaining license notices and package versions. Generate boot mappings from a bounded storage inventory, and allow long SSH diagnostics. Hardware boot qualification remains pending. --- tundra/tests/test_storage_layout.py | 37 +++++++++++++++++++ tundra/tests/test_system_image.py | 27 ++++++++++++++ tundra/tools/build-rescue-boot.py | 9 +++-- tundra/tools/build-system-image.py | 35 ++++++++++++++++++ tundra/tools/inventory-storage.py | 55 +++++++++++++++++++++++++++++ tundra/tools/rescue-ssh.py | 2 +- tundra/tools/storage_layout.py | 32 +++++++++++++++++ 7 files changed, 193 insertions(+), 4 deletions(-) create mode 100644 tundra/tests/test_storage_layout.py create mode 100644 tundra/tools/inventory-storage.py create mode 100644 tundra/tools/storage_layout.py diff --git a/tundra/tests/test_storage_layout.py b/tundra/tests/test_storage_layout.py new file mode 100644 index 00000000..88446726 --- /dev/null +++ b/tundra/tests/test_storage_layout.py @@ -0,0 +1,37 @@ +# SPDX-License-Identifier: MPL-2.0 +from pathlib import Path +import sys +import unittest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1]/'tools')) +from storage_layout import mount_table + + +class StorageLayoutTests(unittest.TestCase): + def layout(self): + return dict(schema=2, device='/dev/mmcblk0p72', partitionLabel='userdata', + filesystem='ext4', backingDevice='259:40', size=53648801280, + table='0 104448000 linear 259:40 329728', firstMiBSha256='a'*64) + + def test_supported_inventory_uses_partition_path(self): + self.assertEqual(mount_table(self.layout()), + '0 104448000 linear /dev/mmcblk0p72 329728') + + def test_different_inventory_offset_is_not_replaced_with_developer_offset(self): + layout=self.layout(); layout['table']='0 104000000 linear 259:40 8192' + self.assertEqual(mount_table(layout), '0 104000000 linear /dev/mmcblk0p72 8192') + + def test_rejects_overflow_other_partition_and_multiple_extents(self): + for patch in ({'table':'0 104448000 linear 259:40 999999999'}, + {'device':'/dev/mmcblk0p1'}, {'filesystem':'crypto_LUKS'}, + {'backingDevice':'259:41'}, + {'table':'0 104448000 linear 259:40 329728\n1 2 linear 259:40 3'}, + {'firstMiBSha256':'not-a-hash'}): + with self.subTest(patch=patch), self.assertRaises(ValueError): + mount_table(self.layout() | patch) + + def test_legacy_inventory_remains_supported_but_not_generalized(self): + layout=self.layout();layout.pop('schema') + self.assertIn('329728',mount_table(layout)) + layout['table']='0 104000000 linear 259:40 8192' + with self.assertRaises(ValueError):mount_table(layout) diff --git a/tundra/tests/test_system_image.py b/tundra/tests/test_system_image.py index a2cebe15..b1931d19 100644 --- a/tundra/tests/test_system_image.py +++ b/tundra/tests/test_system_image.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: MPL-2.0 """Exercise ownership/format safeguards without writing to a real device.""" import importlib.util +import json import os from pathlib import Path import tempfile @@ -14,6 +15,32 @@ spec.loader.exec_module(image) class SystemImageTests(unittest.TestCase): + def test_distribution_omits_working_documents_but_keeps_component_licenses(self): + with tempfile.TemporaryDirectory() as temporary: + root=Path(temporary); product=root/'product'; reference=root/'reference' + (product/'component').mkdir(parents=True);reference.mkdir() + (product/'README.md').write_text('Internal working document') + (product/'component/LICENSE.md').write_text('Original upstream notice') + (reference/'manifest.json').write_text(json.dumps({'packages':{ + 'example':{'binary:Package':'example','Version':'1', + 'source:Package':'example','source:Version':'1', + 'localPath':'private/path'}}})) + image.prepare_distribution_metadata(product,reference) + self.assertFalse((product/'README.md').exists()) + self.assertEqual((product/'component/LICENSE').read_text(),'Original upstream notice') + inventory=json.loads((product/'runtime-packages.json').read_text()) + self.assertEqual(inventory['packages'][0]['binary:Package'],'example') + self.assertNotIn('localPath',inventory['packages'][0]) + self.assertTrue((product/'NOTICE').is_file()) + + def test_conflicting_project_license_is_not_overwritten(self): + with tempfile.TemporaryDirectory() as temporary: + product=Path(temporary) + (product/'LICENSE').write_text('Keep this upstream notice') + with self.assertRaisesRegex(ValueError,'existing notice'): + image.prepare_distribution_metadata(product,Path('/unused')) + self.assertEqual((product/'LICENSE').read_text(),'Keep this upstream notice') + def test_pack_refuses_real_ownership_changes_outside_fakeroot(self): with patch.dict(os.environ, {}, clear=True), patch.object(image.os, 'lchown') as chown: with self.assertRaisesRegex(RuntimeError, 'fakeroot'): diff --git a/tundra/tools/build-rescue-boot.py b/tundra/tools/build-rescue-boot.py index f2c44c2c..aafd6d97 100644 --- a/tundra/tools/build-rescue-boot.py +++ b/tundra/tools/build-rescue-boot.py @@ -16,6 +16,7 @@ import time from cryptography.hazmat.primitives.asymmetric import rsa, ec from cryptography.hazmat.primitives import serialization from ramdisk_reference import read_newc, select_helpers +from storage_layout import mount_table from tundra import ROOT, cpio, checked_source, digest, save, linker @@ -67,8 +68,7 @@ def main(): out=args.output.resolve();out.mkdir(parents=True,exist_ok=False) plan,_=load_boot().checked_plan() layout=json.loads(args.layout.read_text()) - if layout['device']!='/dev/mmcblk0p72' or layout['table']!='0 104448000 linear 259:40 329728' or int(layout['size'])!=53648801280: - raise ValueError('Recipe only supports the inventoried read-only Sargo layout') + storage_table = mount_table(layout) source=Path(plan['backupDirectory'])/'boot_a.img' packager=checked_source('aosp-mkbootimg') unpack=out/'original' @@ -102,7 +102,10 @@ def main(): (out/'known_hosts').write_text('[10.15.19.82]:2222 '+hostpublic+'\n[127.0.0.1]:2222 '+hostpublic+'\n') init=(ROOT/'boot/rescue-init.sh').read_bytes().replace(b'# No LVM activation', ('EXPECTED_STORAGE_HASH='+shlex.quote(layout['firstMiBSha256'])+'\n# No LVM activation').encode()) - sources=[Path(__file__),ROOT/'boot/rescue-init.sh',ROOT/'tools/ramdisk_reference.py'] + init=init.replace(b"'0 104448000 linear /dev/mmcblk0p72 329728'", + shlex.quote(storage_table).encode()) + sources=[Path(__file__),ROOT/'boot/rescue-init.sh',ROOT/'tools/ramdisk_reference.py', + ROOT/'tools/storage_layout.py'] if state: # Keep the existing layout intact; only /tundra/data/.ext4 is used # for Tundra state. Opening its containing filesystem RW is necessary. diff --git a/tundra/tools/build-system-image.py b/tundra/tools/build-system-image.py index b8700fc9..709199a0 100644 --- a/tundra/tools/build-system-image.py +++ b/tundra/tools/build-system-image.py @@ -21,6 +21,39 @@ def digest(path): return hashlib.file_digest(stream, 'sha256').hexdigest() +def prepare_distribution_metadata(product, reference): + """Keep license notices, but leave working documents out of the image.""" + removed = [] + for path in sorted(product.rglob('*')): + if not path.is_file() or path.is_symlink(): + continue + if path.suffix.lower() != '.md': + continue + if path.stem.lower() in ('license', 'copying', 'notice'): + target = path.with_suffix('') + if target.exists() and target.read_bytes() != path.read_bytes(): + raise ValueError('Conflicting license notices: '+str(path)) + if not target.exists(): + path.rename(target) + else: + path.unlink() + else: + removed.append(str(path.relative_to(product))) + path.unlink() + for name in ('LICENSE', 'NOTICE'): + if (product/name).exists() and (product/name).read_bytes() != (PROJECT/name).read_bytes(): + raise ValueError('Refusing to replace an existing notice: '+name) + shutil.copy2(PROJECT/name, product/name) + manifest = json.loads((reference/'manifest.json').read_text()) + packages = [{key: info[key] for key in + ('binary:Package', 'Version', 'Architecture', 'source:Package', 'source:Version') + if key in info} for info in manifest['packages'].values()] + (product/'runtime-packages.json').write_text( + json.dumps({'schema': 1, 'packages': packages}, indent=2)+'\n') + return {'internalMarkdownRemoved': removed, 'packageCount': len(packages), + 'licenseNoticesRetained': True} + + def module(name, path): spec = importlib.util.spec_from_file_location(name, path) result = importlib.util.module_from_spec(spec) @@ -84,6 +117,7 @@ def main(): shutil.copytree(ROOT/'droidian', product/'trial', ignore=shutil.ignore_patterns('__pycache__')) for name in ('libEGL_vulpes_hybris.so', 'manifest.json'): shutil.copy2(args.product/'trial/graphics'/name, product/'trial/graphics'/name) + distribution = prepare_distribution_metadata(product, args.reference) for name in ('home/droidian', 'etc/dbus-1', 'reference'): (root/name).mkdir(parents=True, exist_ok=True) shutil.copy2(android, root/'reference/android-rootfs.img') @@ -116,6 +150,7 @@ RemainAfterExit=yes 'androidSha256': ANDROID_SHA, 'engine': lock, 'release': json.loads((product/'release.json').read_text()), 'staging': staging, 'filesystemCheckPassed': True, + 'distributionContents': distribution, 'flashReady': False, 'hardwareBootTested': False, 'scope': 'Self-contained userspace; vendor/firmware partitions still required; RAM boot candidate', 'sourceHashes': {str(p.relative_to(PROJECT)): digest(p) for p in diff --git a/tundra/tools/inventory-storage.py b/tundra/tools/inventory-storage.py new file mode 100644 index 00000000..f26417f7 --- /dev/null +++ b/tundra/tools/inventory-storage.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MPL-2.0 +"""Inventory an existing Sargo ext4/LVM mount without changing any storage.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import stat +import subprocess + +from storage_layout import mount_table + + +def command(*args): + return subprocess.check_output(args, text=True, timeout=15, + env={**os.environ, 'PATH': '/usr/sbin:/usr/bin:/sbin:/bin', 'LC_ALL': 'C'}).strip() + + +def inventory(mountpoint): + device = Path('/dev/mmcblk0p72') + if not stat.S_ISBLK(device.stat().st_mode): + raise ValueError('Sargo userdata block device is missing') + label = command('blkid', '-s', 'PARTLABEL', '-o', 'value', str(device)) + if label != 'userdata': + raise ValueError('Unexpected partition label') + mounted = json.loads(command('findmnt', '--json', '--mountpoint', mountpoint, + '--output', 'SOURCE,FSTYPE'))['filesystems'] + if len(mounted) != 1 or mounted[0]['fstype'] != 'ext4': + raise ValueError('Expected one existing ext4 mount') + source = mounted[0]['source'] + if not source.startswith('/dev/mapper/'): + raise ValueError('Only the existing single-extent LVM layout is supported') + table = command('dmsetup', 'table', source) + with device.open('rb', buffering=0) as stream: + first = stream.read(1048576) + if len(first) != 1048576: + raise ValueError('Cannot read userdata identity') + layout = { + 'schema': 2, 'device': str(device), 'partitionLabel': label, + 'filesystem': mounted[0]['fstype'], 'table': table, + 'backingDevice': Path('/sys/class/block/mmcblk0p72/dev').read_text().strip(), + 'size': int(command('blockdev', '--getsize64', str(device))), + 'firstMiBSha256': hashlib.sha256(first).hexdigest(), + } + mount_table(layout) + return layout + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--mountpoint', required=True, + help='Existing root mount, or /reference/installed under Tundra') + args = parser.parse_args() + print(json.dumps(inventory(args.mountpoint), indent=2)) diff --git a/tundra/tools/rescue-ssh.py b/tundra/tools/rescue-ssh.py index 88c5c92e..957b7faa 100644 --- a/tundra/tools/rescue-ssh.py +++ b/tundra/tools/rescue-ssh.py @@ -24,7 +24,7 @@ def main(): if not (folder/'boot-report.json').is_file(): raise ValueError('This image has not been sent with the checked boot tool') ssh=['ssh','-F','/dev/null','-o','BatchMode=yes','-o','IdentitiesOnly=yes', - '-o','ConnectTimeout=5','-o','ServerAliveInterval=5','-o','ServerAliveCountMax=6', + '-o','ConnectTimeout=5','-o','ServerAliveInterval=5','-o','ServerAliveCountMax='+('60' if args.stream else '6'), '-o','StrictHostKeyChecking=yes','-o','HostKeyAlgorithms=+ssh-rsa', '-o','UserKnownHostsFile='+str(folder/'known_hosts'),'-i',str(folder/'client-key'), '-p','2222','root@10.15.19.82'] diff --git a/tundra/tools/storage_layout.py b/tundra/tools/storage_layout.py new file mode 100644 index 00000000..2827bd14 --- /dev/null +++ b/tundra/tools/storage_layout.py @@ -0,0 +1,32 @@ +# SPDX-License-Identifier: MPL-2.0 +"""Validate an inventoried, single-extent Sargo userdata mapping.""" +import re + + +def mount_table(layout): + device = layout.get('device') + if device != '/dev/mmcblk0p72': + raise ValueError('Expected Sargo userdata, not another partition') + if not re.fullmatch(r'[0-9a-f]{64}', layout.get('firstMiBSha256', '')): + raise ValueError('Missing userdata identity checksum') + match = re.fullmatch(r'0 ([0-9]+) linear ([0-9]+:[0-9]+) ([0-9]+)', + layout.get('table', '')) + if not match: + raise ValueError('Only a single linear extent is supported') + sectors, backing, offset = int(match[1]), match[2], int(match[3]) + size = int(layout['size']) + if size <= 0 or size % 512 or sectors <= 0 or offset < 2048: + raise ValueError('Invalid userdata extent geometry') + if (sectors + offset) * 512 > size: + raise ValueError('Extent exceeds userdata') + if layout.get('schema') == 2: + if (layout.get('partitionLabel') != 'userdata' or + layout.get('filesystem') != 'ext4' or + layout.get('backingDevice') != backing): + raise ValueError('Mapping must be ext4 on the inventoried userdata device') + elif not (layout['table'] == '0 104448000 linear 259:40 329728' + and size == 53648801280): + raise ValueError('A new layout requires a schema 2 inventory') + # Kernel major/minor numbers can change between boots. The partition path + # is stable with this Sargo kernel, and its first MiB is checked before use. + return f'0 {sectors} linear {device} {offset}'