(WIP) Rpi arm64 support

fablib/installer.py

@@ -11,12 +11,10 @@
 import os
 from os.path import join, exists, basename
 import shutil
-from typing import (
-        Iterable, Optional, Dict, Tuple, List, TextIO, IO, AnyStr, cast
-)
+from typing import Iterable, Optional, TextIO, IO, AnyStr, cast
 
 import hashlib
-import debian
+from debian import debfile
 
 from chroot import Chroot
 from fablib import common
@@ -108,7 +106,7 @@ def revert(self) -> None:
 class Installer:
     def __init__(
             self, chroot_path: str,
-            environ: Optional[Dict[str, str]]=None
+            environ: dict[str, str] = None
     ):
         if environ is None:
             environ = {}
@@ -118,7 +116,7 @@ def __init__(
         self.chroot = Chroot(chroot_path, environ=env)
 
     @staticmethod
-    def _get_packages_priority(packages: List[str]) -> Tuple[List[str], List[str]]:
+    def _get_packages_priority(packages: list[str]) -> tuple[list[str], list[str]]:
         """high priority packages must be installed before regular packages
            APT should handle this, but in some circumstances it chokes...
         """
@@ -136,9 +134,9 @@ def _get_packages_priority(packages: List[str]) -> Tuple[List[str], List[str]]:
         return high, regular
 
     def _install(
-            self, packages: List[str],
-            ignore_errors: Optional[List[str]]=None,
-            extra_apt_args: Optional[List[str]]=None) -> None:
+            self, packages: list[str],
+            ignore_errors: list[str] = None,
+            extra_apt_args: list[str] = None) -> None:
 
         if ignore_errors is None:
             ignore_errors = []
@@ -161,7 +159,7 @@ def _install(
             "#!/bin/sh",
             "echo",
             'echo "Warning: Deferring update-initramfs $@"',
-            'echo "update-initramfs $@" >> /%s' % defer_log,
+            f'echo "update-initramfs $@" >> /{defer_log}'
         ]
         fake_update_initramfs = RevertibleScript(
             join(self.chroot.path, "usr/sbin/update-initramfs"), lines
@@ -177,7 +175,7 @@ def _install(
                         f"apt-get {' '.join((args + packages))}")
                 if apt_return_code != 0:
 
-                    def get_last_log(path: str) -> List[str]:
+                    def get_last_log(path: str) -> list[str]:
                         log = []
                         with open(path) as fob:
                             for line in fob:
@@ -190,7 +188,7 @@ def get_last_log(path: str) -> List[str]:
                         log.reverse()
                         return log
 
-                    def get_errors(log: List[str], error_str: str) -> List[str]:
+                    def get_errors(log: list[str], error_str: str) -> list[str]:
                         errors = []
                         for line in reversed(log):
                             if line == error_str:
@@ -222,13 +220,11 @@ def get_errors(log: List[str], error_str: str) -> List[str]:
                     errors = set(errors) - set(ignore_errors)
 
                     if ignored_errors:
-                        print(
-                            "Warning: ignoring package installation errors (%s)"
-                            % " ".join(ignored_errors)
-                        )
+                        print(f"Warning: ignoring package installation errors"
+                              f" ({' '.join(ignored_errors)})")
 
                     if errors:
-                        for error in error:
+                        for error in errors:
                             common.error(error)
                         raise Error('package installation errors')
 
@@ -253,15 +249,15 @@ def get_errors(log: List[str], error_str: str) -> List[str]:
             os.remove(defer_log)
 
     def install(
-            self, packages: List[str],
-            ignore_errors: Optional[List[str]]=None) -> None:
+            self, packages: list[str],
+            ignore_errors: list[str] = None) -> None:
         raise NotImplementedError()
 
 
 class PoolInstaller(Installer):
     def __init__(
             self, chroot_path: str, pool_path: str,
-            arch: str, environ: Optional[Dict[str, str]]=None):
+            arch: str, environ: dict[str, str] = None):
         super(PoolInstaller, self).__init__(chroot_path, environ)
 
         from pool_lib import Pool
@@ -270,7 +266,7 @@ def __init__(
         self.arch = arch
 
     @staticmethod
-    def _get_package_index(packagedir: str) -> List[str]:
+    def _get_package_index(packagedir: str) -> list[str]:
         def filesize(path: str) -> str:
             return str(os.stat(path).st_size)
 
@@ -288,7 +284,7 @@ def sha256sum(path: str) -> str:
             # dl_path would best be calculated; but we don't have access to chroot_path here...
             dl_path = os.path.join("var/cache/apt/archives", package)
             if path.endswith(".deb"):
-                control = debian.debfile.DebFile(path).debcontrol()
+                control = debfile.DebFile(path).debcontrol()
                 for field in list(control.keys()):
                     index.append(field + ": " + control[field])
 
@@ -301,8 +297,8 @@ def sha256sum(path: str) -> str:
         return index
 
     def install(
-            self, packages: List[str],
-            ignore_errors: Optional[List[str]]=None
+            self, packages: list[str],
+            ignore_errors: list[str] = None
     ) -> None:
         """install packages into chroot via pool"""
 
@@ -321,7 +317,7 @@ def install(
         print("deb file:/// local debs", file=cast(TextIO, sources_list))
         sources_list.close()
 
-        index_file = "_dists_local_debs_binary-%s_Packages" % self.arch
+        index_file = f"_dists_local_debs_binary-{self.arch}_Packages"
         index_path = join(self.chroot.path, "var/lib/apt/lists", index_file)
         index = self._get_package_index(packagedir)
         with open(index_path, "w") as fob:
@@ -335,15 +331,15 @@ def install(
 class LiveInstaller(Installer):
     def __init__(
             self, chroot_path: str,
-            apt_proxy: Optional[str]=None,
-            environ: Optional[Dict[str, str]]=None):
+            apt_proxy: str = None,
+            environ: dict[str, str] = None):
         super(LiveInstaller, self).__init__(chroot_path, environ)
 
         self.apt_proxy = apt_proxy
 
     def install(
-            self, packages: List[str],
-            ignore_errors: Optional[List[str]]=None) -> None:
+            self, packages: list[str],
+            ignore_errors: list[str] = None) -> None:
         """install packages into chroot via live apt"""
         if ignore_errors is None:
             ignore_errors = []

share/product.mk

@@ -25,6 +25,22 @@ DEBIAN = $(shell [ $(DISTRO) = 'debian' ] && echo 'y')
 FAB_ARCH = $(shell dpkg --print-architecture)
 I386 = $(shell [ $(FAB_ARCH) = 'i386' ] && echo 'y')
 AMD64 = $(shell [ $(FAB_ARCH) = 'amd64' ] && echo 'y')
+ARM64 = $(shell [ $(FAB_ARCH) = 'arm64' ] && echo 'y')
+
+ifeq ($(I386),y)
+ARCH_FAMILY=x86
+endif
+ifeq ($(AMD64),y)
+ARCH_FAMILY=x86
+endif
+ifeq ($(ARM64),y)
+ARCH_FAMILY=arm
+# NONFREE is used to get raspi-firmware and firmware-brcm80211
+NONFREE=1
+endif
+ifndef ARCH_FAMILY
+$(error unsupported architecture)
+endif
 
 ifdef FAB_POOL
 FAB_POOL_PATH=$(FAB_PATH)/pools/$(CODENAME)
@@ -45,7 +61,7 @@ endif
 
 COMMON_PATCHES := turnkey.d $(COMMON_PATCHES)
 
-CONF_VARS_BUILTIN ?= FAB_ARCH FAB_HTTP_PROXY I386 AMD64 RELEASE DISTRO CODENAME DEBIAN UBUNTU KERNEL DEBUG CHROOT_ONLY
+CONF_VARS_BUILTIN ?= FAB_ARCH FAB_HTTP_PROXY I386 AMD64 ARM64 RELEASE DISTRO CODENAME DEBIAN UBUNTU KERNEL DEBUG CHROOT_ONLY
 
 define filter-undefined-vars
 	$(foreach var,$1,$(if $($(var)), $(var)))
@@ -136,8 +152,12 @@ endef
 ifdef CHROOT_ONLY
 all: root.sandbox
 else
+ifeq ($(ARCH_FAMILY),arm)
+all: $O/product.img
+else
 all: $O/product.iso
 endif
+endif
 
 define mount-deck
 	@(deck $1 > /dev/null 2>&1) && echo deck $1 || true
@@ -173,6 +193,7 @@ define help/body
 	@echo '  CONF_VARS                  $(value CONF_VARS)'
 	@echo
 	@echo '  FAB_ARCH                   $(value FAB_ARCH)'
+	@echo '  ARCH_FAMILY                $(value ARCH_FAMILY)'
 	@echo '  FAB_POOL                   $(value FAB_POOL)'
 	@echo '  FAB_POOL_PATH              $(value FAB_POOL_PATH)'
 	@echo '  FAB_PLAN_INCLUDE_PATH      $(value FAB_PLAN_INCLUDE_PATH)/'
@@ -222,7 +243,12 @@ define help/body
 	@echo '# remake target and the targets that depend on it'
 	@echo '$$ rm $(value STAMPS_DIR)/<target>; make <target>'
 	@echo
-	@echo '# build a target (default: product.iso)'
+	@if [ "$(ARCH_FAMILY)" = "arm" ]; \
+	then \
+	echo '# build a target (default: product.img)'; \
+	else \
+	echo '# build a target (default: product.iso)'; \
+	fi
 	@echo '$$ make [target] [O=path/to/build/dir]'
 	@echo '  redeck        # deck unmounted input/output decks (e.g., after reboot)'
 	@echo
@@ -231,19 +257,24 @@ define help/body
 	@echo '  root.spec     # the spec from which root.build is built (I.e., resolved plan)'
 	@echo '  root.build    # created by applying the root.spec to the bootstrap'
 	@echo '  root.patched  # deck root.build and apply the root overlay and removelist'
-    @echo
+	@echo
 	@echo '  root.sandbox  # changes (e.g., manual prototyping) inside the copy-on-write sandbox'
 	@echo '                # saved as a separate, temporary cdroot squashfs overlay'
 	@echo
 endef
 
 ifndef CHROOT_ONLY
+ifeq ($(ARCH_FAMILY),arm)
+help/body += ;\
+	echo '  product.img   \# product img for raspberry pi 4';
+else
 help/body += ;\
 	echo '  cdroot        \# created by squashing root.patched into cdroot template + overlay'; \
 	echo '  product.iso   \# product ISO created from the cdroot'; \
 	echo; \
 	echo '  updated-initramfs \# rebuild product with updated initramfs' 
 endif
+endif
 
 help:
 	$(help/pre)
@@ -255,7 +286,7 @@ define clean/body
 	$(call remove-deck, $O/root.patched)
 	$(call remove-deck, $O/root.build)
 	$(call remove-deck, $O/bootstrap)
-	-rm -rf $O/root.spec $O/cdroot $O/product.iso $O/log $(STAMPS_DIR)
+	-rm -rf $O/root.spec $O/cdroot $O/product.iso $O/sdroot $O/product.img $O/product.img.xz $O/log $(STAMPS_DIR)
 endef
 
 clean:
@@ -303,7 +334,7 @@ define run-conf-scripts
 	if [ -n "$(wildcard $1/*)" ]; then \
 		echo "\$$(call $0,$1)"; \
 	fi; \
-	for script in $1/*; do \
+	for script in $1/* $1/$(ARCH_FAMILY).d/*; do \
 		[ -f "$$script" ] && [ -x "$$script" ] || continue; \
 		args_path=$(strip $1)/args/$$(echo $$(basename $$script) | sed 's/^[^a-zA-Z]*//'); \
 		args="$$([ -f $$args_path ] && (cat $$args_path | sed 's/#.*//'))"; \
@@ -337,7 +368,8 @@ define root.patched/body
 	# apply the common overlays
 	$(foreach overlay,$(_COMMON_OVERLAYS),
 	  @if echo $(overlay) | grep -q '\.d$$'; then \
-	  	for d in $(overlay)/*; do \
+		for d in $(overlay)/* $(overlay)/$(ARCH_FAMILY).d/*; do \
+		  if echo $$d | grep -q '\.d$$'; then continue; fi; \
 		  echo fab-apply-overlay $$d $O/root.patched; \
 		  fab-apply-overlay $$d $O/root.patched; \
 		done; \
@@ -494,6 +526,26 @@ define product.iso/body
 	$(run-isohybrid)
 endef
 
+# target: product.img
+define product.img/body
+	qemu-img create -f raw $O/product.img 2G
+	parted -s $O/product.img mklabel msdos
+	parted -s $O/product.img -- mkpart primary fat32 4MiB 400MiB
+	parted -s $O/product.img -- mkpart primary ext2 400MiB 100%
+	kpartx -asv $O/product.img
+	mkfs -t vfat -n RASPIFIRM /dev/mapper/loop0p1
+	mkfs -t ext4 -L RASPIROOT /dev/mapper/loop0p2
+	mkdir -p $O/sdroot
+	mount /dev/mapper/loop0p2 $O/sdroot
+	mkdir -p $O/sdroot/boot/firmware
+	mount /dev/mapper/loop0p1 $O/sdroot/boot/firmware
+	cp -ax $O/root.sandbox/* $O/sdroot
+	umount $O/sdroot/boot/firmware
+	umount $O/sdroot
+	kpartx -dsv $O/product.img
+	xz -8 -f $O/product.img
+endef
+
 cdroot-dynamic: $(STAMPS_DIR)/root.sandbox
 	$(cdroot-dynamic/pre)
 	$(cdroot-dynamic/body)
@@ -530,6 +582,14 @@ $O/product.iso: $(product.iso/deps) $(product.iso/deps/extra)
 
 product.iso: $O/product.iso
 
+product.img/deps ?= $(STAMPS_DIR)/root.sandbox
+$O/product.img: $(product.img/deps) $(product.img/deps/extra)
+	$(product.img/pre)
+	$(product.img/body)
+	$(product.img/post)
+
+product.img: $O/product.img
+
 # target: updated-initramfs
 define updated-initramfs/body
 	rm -rf $O/product.iso