feat: create firmware skeleton and reproducible build

scripts/generate_web_assets.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+"""Generate deterministic gzip-compressed LittleFS web assets."""
+
+from __future__ import annotations
+
+import argparse
+import gzip
+import os
+import shutil
+from pathlib import Path
+
+
+SUPPORTED_SUFFIXES = {".css", ".html", ".ico", ".js", ".json", ".svg"}
+
+
+def generate_assets(source_directory: Path, output_directory: Path) -> list[Path]:
+    source_directory = source_directory.resolve()
+    output_directory = output_directory.resolve()
+    if not source_directory.is_dir():
+        raise ValueError(f"Web source directory does not exist: {source_directory}")
+    if source_directory == output_directory:
+        raise ValueError("Web source and output directories must differ")
+
+    source_files = sorted(
+        path for path in source_directory.rglob("*")
+        if path.is_file() and path.suffix.lower() in SUPPORTED_SUFFIXES
+    )
+    if not source_files:
+        raise ValueError(f"No supported web assets found in {source_directory}")
+
+    if output_directory.exists():
+        shutil.rmtree(output_directory)
+    output_directory.mkdir(parents=True)
+
+    generated_files: list[Path] = []
+    for source_path in source_files:
+        relative_path = source_path.relative_to(source_directory)
+        output_path = output_directory / f"{relative_path}.gz"
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        with source_path.open("rb") as source, output_path.open("wb") as output:
+            with gzip.GzipFile(
+                filename="",
+                mode="wb",
+                compresslevel=9,
+                fileobj=output,
+                mtime=0,
+            ) as compressed:
+                shutil.copyfileobj(source, compressed)
+        os.utime(output_path, ns=(0, 0))
+        generated_files.append(output_path)
+
+    for directory in sorted(
+        (path for path in output_directory.rglob("*") if path.is_dir()),
+        reverse=True,
+    ):
+        os.utime(directory, ns=(0, 0))
+    os.utime(output_directory, ns=(0, 0))
+
+    return generated_files
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--source", type=Path, default=Path("web"))
+    parser.add_argument("--output", type=Path, default=Path("data"))
+    arguments = parser.parse_args()
+
+    generated_files = generate_assets(arguments.source, arguments.output)
+    for path in generated_files:
+        print(path)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/platformio_build.py

@@ -0,0 +1,93 @@
+"""PlatformIO hooks for deterministic assets and partition size limits."""
+
+from __future__ import annotations
+
+import csv
+import subprocess
+from pathlib import Path
+
+Import("env")
+
+
+PROJECT_DIRECTORY = Path(env.subst("$PROJECT_DIR"))
+BUILD_DIRECTORY = Path(env.subst("$BUILD_DIR"))
+PARTITION_FILE = PROJECT_DIRECTORY / "partitions.csv"
+ASSET_SCRIPT = PROJECT_DIRECTORY / "scripts" / "generate_web_assets.py"
+FILESYSTEM_WRAPPER = PROJECT_DIRECTORY / "scripts" / "reproducible_mklittlefs.py"
+
+
+def parse_size(value: str) -> int:
+    value = value.strip()
+    suffixes = {"K": 1024, "M": 1024 * 1024}
+    if value[-1:].upper() in suffixes:
+        return int(value[:-1], 0) * suffixes[value[-1].upper()]
+    return int(value, 0)
+
+
+def partition_sizes() -> dict[str, int]:
+    sizes: dict[str, int] = {}
+    with PARTITION_FILE.open(newline="", encoding="utf-8") as partition_stream:
+        for row in csv.reader(partition_stream):
+            if not row or row[0].lstrip().startswith("#"):
+                continue
+            if len(row) < 5:
+                raise RuntimeError(f"Invalid partition row: {row}")
+            sizes[row[0].strip()] = parse_size(row[4])
+    return sizes
+
+
+def generate_assets(source, target, env) -> None:
+    command = [
+        env.subst("$PYTHONEXE"),
+        str(ASSET_SCRIPT),
+        "--source",
+        str(PROJECT_DIRECTORY / "web"),
+        "--output",
+        str(PROJECT_DIRECTORY / "data"),
+    ]
+    subprocess.run(command, check=True)
+
+
+def enforce_size(path: Path, partition_name: str) -> None:
+    limits = partition_sizes()
+    if partition_name not in limits:
+        raise RuntimeError(f"Missing required partition: {partition_name}")
+    actual_size = path.stat().st_size
+    maximum_size = limits[partition_name]
+    if actual_size > maximum_size:
+        raise RuntimeError(
+            f"{path.name} is {actual_size} bytes and exceeds the "
+            f"{partition_name} partition limit of {maximum_size} bytes"
+        )
+    print(
+        f"maze32 size check: {path.name} uses {actual_size}/{maximum_size} bytes "
+        f"in partition {partition_name}"
+    )
+
+
+def check_application(source, target, env) -> None:
+    enforce_size(application_image, "factory")
+
+
+def check_filesystem(source, target, env) -> None:
+    enforce_size(filesystem_image, "littlefs")
+
+
+filesystem_image = BUILD_DIRECTORY / "littlefs.bin"
+application_image = BUILD_DIRECTORY / f"{env.subst('$PROGNAME')}.bin"
+real_filesystem_tool = env.subst("$MKFSTOOL")
+
+env["ENV"]["MAZE32_REAL_MKFSTOOL"] = real_filesystem_tool
+env["ENV"]["MAZE32_BUILD_DIR"] = str(BUILD_DIRECTORY)
+env.Replace(MKFSTOOL=str(FILESYSTEM_WRAPPER))
+
+env.AddPreAction("$BUILD_DIR/littlefs.bin", generate_assets)
+env.AddPostAction("$BUILD_DIR/littlefs.bin", check_filesystem)
+env.AddPostAction("$BUILD_DIR/${PROGNAME}.bin", check_application)
+env.AddCustomTarget(
+    "webassets",
+    None,
+    generate_assets,
+    title="Generate deterministic web assets",
+    description="Compress web sources into the LittleFS data directory",
+)

scripts/reproducible_mklittlefs.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+"""Run mklittlefs with a fixed wall clock for reproducible images."""
+
+from __future__ import annotations
+
+import os
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+
+def build_time_override(output_path: Path) -> None:
+    source_path = Path(__file__).with_name("reproducible_time.c")
+    compiler = shutil.which("cc")
+    if compiler is None:
+        raise RuntimeError("A host C compiler named 'cc' is required")
+    subprocess.run(
+        [
+            compiler,
+            "-shared",
+            "-fPIC",
+            "-O2",
+            str(source_path),
+            "-o",
+            str(output_path),
+        ],
+        check=True,
+    )
+
+
+def main() -> int:
+    real_tool = os.environ.get("MAZE32_REAL_MKFSTOOL")
+    build_directory = os.environ.get("MAZE32_BUILD_DIR")
+    if not real_tool or not build_directory:
+        raise RuntimeError("PlatformIO did not provide the filesystem tool environment")
+
+    override_path = Path(build_directory) / "reproducible_time.so"
+    override_path.parent.mkdir(parents=True, exist_ok=True)
+    build_time_override(override_path)
+
+    environment = os.environ.copy()
+    existing_preload = environment.get("LD_PRELOAD")
+    environment["LD_PRELOAD"] = (
+        f"{override_path}:{existing_preload}" if existing_preload else str(override_path)
+    )
+    completed = subprocess.run([real_tool, *sys.argv[1:]], env=environment)
+    return completed.returncode
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/reproducible_time.c

@@ -0,0 +1,37 @@
+#define _GNU_SOURCE
+
+#include <stddef.h>
+#include <sys/syscall.h>
+#include <sys/time.h>
+#include <time.h>
+#include <unistd.h>
+
+static const time_t FIXED_TIME = 0;
+
+time_t time(time_t *result)
+{
+    if (result != NULL) {
+        *result = FIXED_TIME;
+    }
+    return FIXED_TIME;
+}
+
+int gettimeofday(struct timeval *value, void *timezone)
+{
+    (void)timezone;
+    if (value != NULL) {
+        value->tv_sec = FIXED_TIME;
+        value->tv_usec = 0;
+    }
+    return 0;
+}
+
+int clock_gettime(clockid_t clock_id, struct timespec *value)
+{
+    if (clock_id == CLOCK_REALTIME) {
+        value->tv_sec = FIXED_TIME;
+        value->tv_nsec = 0;
+        return 0;
+    }
+    return (int)syscall(SYS_clock_gettime, clock_id, value);
+}

scripts/run_host_tests.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env sh
+set -eu
+
+project_directory=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
+test_directory=$(mktemp -d "${TMPDIR:-/tmp}/maze32-host-tests.XXXXXX")
+trap 'rm -rf "$test_directory"' EXIT HUP INT TERM
+
+cc -std=c11 -Wall -Wextra -Werror -pedantic \
+    -I"$project_directory/components/maze32_platform/include" \
+    "$project_directory/components/maze32_platform/maze32_health.c" \
+    "$project_directory/test/host/test_health.c" \
+    -o "$test_directory/test_health"
+
+"$test_directory/test_health"
+python3 "$project_directory/scripts/verify_web_assets.py"

scripts/verify_web_assets.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+"""Verify deterministic web generation and the checked-in generated assets."""
+
+from __future__ import annotations
+
+import gzip
+import hashlib
+import tempfile
+from pathlib import Path
+
+from generate_web_assets import generate_assets
+
+
+PROJECT_DIRECTORY = Path(__file__).resolve().parent.parent
+
+
+def tree_digest(directory: Path) -> str:
+    digest = hashlib.sha256()
+    for path in sorted(item for item in directory.rglob("*") if item.is_file()):
+        digest.update(path.relative_to(directory).as_posix().encode("utf-8"))
+        digest.update(b"\0")
+        digest.update(path.read_bytes())
+    return digest.hexdigest()
+
+
+def main() -> int:
+    source_directory = PROJECT_DIRECTORY / "web"
+    checked_in_directory = PROJECT_DIRECTORY / "data"
+    with tempfile.TemporaryDirectory(prefix="maze32-assets-") as temporary:
+        temporary_directory = Path(temporary)
+        first = temporary_directory / "first"
+        second = temporary_directory / "second"
+        generate_assets(source_directory, first)
+        generate_assets(source_directory, second)
+
+        first_digest = tree_digest(first)
+        second_digest = tree_digest(second)
+        checked_in_digest = tree_digest(checked_in_directory)
+        if first_digest != second_digest:
+            raise AssertionError("Repeated web-asset generation produced different output")
+        if first_digest != checked_in_digest:
+            raise AssertionError("Generated web assets are stale; regenerate the data directory")
+
+        index_source = (source_directory / "index.html").read_bytes()
+        index_compressed = (first / "index.html.gz").read_bytes()
+        if gzip.decompress(index_compressed) != index_source:
+            raise AssertionError("Compressed index does not reproduce its source")
+
+    print(f"web asset reproducibility: PASS ({first_digest})")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())