From 4483f4cdfbb411618340f643b2123b1ede8c12ca Mon Sep 17 00:00:00 2001 From: Cyril Brulebois Date: Sat, 28 Aug 2021 04:01:30 +0200 Subject: [PATCH 01/12] Generate recipes using a Python script instead of multiplying sed calls. This is a proof of concept rather than an ideal, final situation. It can be used this way: for v in 1 2 3 4 ; do for s in buster bullseye; do ./generate-recipe.py $v $s done done and it has been verified to produce very similar results compared to the existing many-sed approach. Differences are as follows: - Missing newline after some backports stanza, due to the removal of the other APT line. There's already MR#51 that aims at fixing some newline-related issues anyway, so this can be addressed separately. - Less schizophrenia in the generated sources.list for buster/4, as we are now only showing a reason for enabling the backports, instead of starting by explaining why backports are disabled by default. - Dropping APT::Default-Release = buster in the buster/4 case, which is no longer needed as we are pulling things from buster-backports rather than pulling them from unstable (see 57e90df103). - No longer trying to fix the firmware package name by throwing a broken sed at rpi-reconfigure-raspi-firmware.service in the buster/4 case: the syntax was buggy and fixing it would have made us try to replace raspi-firmware with raspi-firmware/buster-backports, while the correct thing to do is to not touch it in the first place (raspi-firmware is the correct name for the firmware package, pulled from buster-backports). As a side effect, this transforms the existing __EXTRA_SHELL_CMDS__ into a slightly more explicit __EXTRA_ROOT_SHELL_CMDS__ which now has its __EXTRA_CHROOT_SHELL_CMDS__ twin. That's the entry point that was missing and made 45cb5619d4 necessary in the past. --- generate-recipe.py | 160 +++++++++++++++++++++++++++++++++++++++++++++ raspi_master.yaml | 10 ++- 2 files changed, 164 insertions(+), 6 deletions(-) create mode 100755 generate-recipe.py diff --git a/generate-recipe.py b/generate-recipe.py new file mode 100755 index 0000000..f34a75c --- /dev/null +++ b/generate-recipe.py @@ -0,0 +1,160 @@ +#!/usr/bin/python3 + +import re +import sys + +# pylint: disable=invalid-name + +### Sanity/usage checks + +if len(sys.argv) != 3: + print("E: need 2 arguments", file=sys.stderr) + sys.exit(1) + +version = sys.argv[1] +if version not in ["1", "2", "3", "4"]: + print("E: unsupported version %s" % version, file=sys.stderr) + sys.exit(1) + +suite = sys.argv[2] +if suite not in ['buster', 'bullseye']: + print("E: unsupported suite %s" % suite, file=sys.stderr) + sys.exit(1) +target_yaml = 'raspi_%s_%s.yaml' % (version, suite) + + +### Setting variables based on suite and version starts here + +# Arch, kernel, DTB: +if version == '1': + arch = 'armel' + linux = 'linux-image-rpi' + dtb = '/usr/lib/linux-image-*-rpi/bcm*rpi-*.dtb' +elif version == '2': + arch = 'armhf' + linux = 'linux-image-armmp' + dtb = '/usr/lib/linux-image-*-armmp/bcm*rpi*.dtb' +elif version in ['3', '4']: + arch = 'arm64' + linux = 'linux-image-arm64' + dtb = '/usr/lib/linux-image-*-arm64/broadcom/bcm*rpi*.dtb' + +# APT and default firmware (name + handling) +if suite == 'buster': + security_suite = '%s/updates' % suite + raspi_firmware = 'raspi3-firmware' + fix_firmware = True +else: + security_suite = '%s-security' % suite + raspi_firmware = 'raspi-firmware' + fix_firmware = False + +# Extra wireless firmware: +if version != '2': + wireless_firmware_maybe = '- firmware-brcm80211' +else: + wireless_firmware_maybe = '' + +# Serial console: +if version in ['1', '2']: + serial = 'ttyAMA0,115200' +elif version in ['3', '4']: + serial = 'ttyS1,115200' + +# Pi 4 on buster requires some backports: +backports_enable = False +backports_suite = '%s-backports' % suite +if version == '4' and suite == 'buster': + backports_enable = "# raspi 4 needs kernel and firmware newer than buster's" + linux = '%s/%s' % (linux, backports_suite) + raspi_firmware = 'raspi-firmware/%s' % backports_suite + wireless_firmware_maybe = '- firmware-brcm80211/%s' % backports_suite + fix_firmware = False + +# CMA fixup: +extra_chroot_shell_cmds = [] +if version == '4': + extra_chroot_shell_cmds = [ + "sed -i 's/cma=64M //' /boot/firmware/cmdline.txt", + "sed -i 's/cma=$CMA //' /etc/kernel/postinst.d/z50-raspi-firmware", + ] + +# XXX: The disparity between suite seems to be a bug, pick a naming +# and stick to it! +# +# Hostname: +if suite == 'buster': + hostname = 'rpi%s' % version +else: + hostname = 'rpi_%s' % version + +# Nothing yet! +extra_root_shell_cmds = [] + + +### The following prepares substitutions based on variables set earlier + +# Commands to fix the firmware name in the systemd unit: +if fix_firmware: + fix_firmware_cmds = ['sed -i s/raspi-firmware/raspi3-firmware/ ${ROOT?}/etc/systemd/system/rpi-reconfigure-raspi-firmware.service'] +else: + fix_firmware_cmds = [] + +# Enable backports with a reason, or add commented-out entry: +if backports_enable: + backports_stanza = """ +%s +deb http://deb.debian.org/debian/ %s main contrib non-free +""" % (backports_enable, backports_suite) +else: + backports_stanza = """ +# Backports are _not_ enabled by default. +# Enable them by uncommenting the following line: +# deb http://deb.debian.org/debian %s main contrib non-free +""" % backports_suite + + +### Write results: + +def align_replace(text, pattern, replacement): + """ + This helper lets us keep the indentation of the matched pattern + with the upcoming replacement, across multiple lines. Naive + implementation, please make it more pythonic! + """ + lines = text.splitlines() + for i, line in enumerate(lines): + m = re.match(r'^(\s+)%s' % pattern, line) + if m: + indent = m.group(1) + del lines[i] + for r in replacement: + lines.insert(i, '%s%s' % (indent, r)) + i = i + 1 + break + return '\n'. join(lines) + '\n' + + +with open('raspi_master.yaml', 'r') as in_file: + with open(target_yaml, 'w') as out_file: + in_text = in_file.read() + out_text = in_text \ + .replace('__RELEASE__', suite) \ + .replace('__SECURITY_SUITE__', security_suite) \ + .replace('__FIRMWARE_PKG__', raspi_firmware) \ + .replace('__WIRELESS_FIRMWARE_PKG_MAYBE__', wireless_firmware_maybe) \ + .replace('__SERIAL_CONSOLE__', serial) \ + .replace('__LINUX_IMAGE__', linux) \ + .replace('__DTB__', dtb) \ + .replace('__HOST__', hostname) \ + .replace('__ARCH__', arch) + + out_text = align_replace(out_text, '__FIX_FIRMWARE_PKG_NAME__', fix_firmware_cmds) + out_text = align_replace(out_text, '__EXTRA_ROOT_SHELL_CMDS__', extra_root_shell_cmds) + out_text = align_replace(out_text, '__EXTRA_CHROOT_SHELL_CMDS__', extra_chroot_shell_cmds) + out_text = align_replace(out_text, '__BACKPORTS__', backports_stanza.splitlines()) + + # Try not to keep lines where the placeholder was replaced + # with nothing at all: + filtered = [x for x in out_text.splitlines() if not re.match(r'^\s+$', x)] + out_file.write('\n'.join(filtered) + "\n") diff --git a/raspi_master.yaml b/raspi_master.yaml index d8ffae6..771ed8e 100644 --- a/raspi_master.yaml +++ b/raspi_master.yaml @@ -53,10 +53,7 @@ steps: contents: | deb http://deb.debian.org/debian __RELEASE__ main contrib non-free deb http://security.debian.org/debian-security __SECURITY_SUITE__ main contrib non-free - # Backports are _not_ enabled by default. - # Enable them by uncommenting the following line: - # deb http://deb.debian.org/debian __RELEASE__-backports main contrib non-free - __OTHER_APT_ENABLE__ + __BACKPORTS__ unless: rootfs_unpacked - copy-file: /etc/initramfs-tools/hooks/rpi-resizerootfs @@ -84,7 +81,7 @@ steps: - wpasupplicant - __FIRMWARE_PKG__ - __LINUX_IMAGE__ - __EXTRA_PKGS__ + __WIRELESS_FIRMWARE_PKG_MAYBE__ tag: / unless: rootfs_unpacked @@ -121,7 +118,7 @@ steps: ln -s /etc/systemd/system/rpi-generate-ssh-host-keys.service "${ROOT?}/etc/systemd/system/multi-user.target.requires/rpi-generate-ssh-host-keys.service" rm -f "${ROOT?}"/etc/ssh/ssh_host_*_key* - __EXTRA_SHELL_CMDS__ + __EXTRA_ROOT_SHELL_CMDS__ root-fs: / # Copy the relevant device tree files to the boot partition @@ -145,6 +142,7 @@ steps: shell: | sed -i 's/^/console=__SERIAL_CONSOLE__ /' /boot/firmware/cmdline.txt sed -i 's/.dev.mmcblk0p2/LABEL=RASPIROOT/' /boot/firmware/cmdline.txt + __EXTRA_CHROOT_SHELL_CMDS__ # TODO(https://github.com/larswirzenius/vmdb2/issues/24): remove once vmdb # clears /etc/resolv.conf on its own. From 433100f5cbf3f2ae71e113116c4a7d466b661c71 Mon Sep 17 00:00:00 2001 From: Cyril Brulebois Date: Sat, 28 Aug 2021 05:20:39 +0200 Subject: [PATCH 02/12] Standardize firmware handling. Group raspi-firmware and firmware-brcm80211 together, and make the firmware package a regular list item in the master YAML file (making editors happy about it). Of course, this means that in all generated recipes, linux-image and raspi*-firmware switch places. --- generate-recipe.py | 16 +++++++++------- raspi_master.yaml | 4 ++-- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/generate-recipe.py b/generate-recipe.py index f34a75c..7e2caac 100755 --- a/generate-recipe.py +++ b/generate-recipe.py @@ -51,9 +51,9 @@ else: # Extra wireless firmware: if version != '2': - wireless_firmware_maybe = '- firmware-brcm80211' + wireless_firmware = 'firmware-brcm80211' else: - wireless_firmware_maybe = '' + wireless_firmware = '' # Serial console: if version in ['1', '2']: @@ -68,7 +68,7 @@ if version == '4' and suite == 'buster': backports_enable = "# raspi 4 needs kernel and firmware newer than buster's" linux = '%s/%s' % (linux, backports_suite) raspi_firmware = 'raspi-firmware/%s' % backports_suite - wireless_firmware_maybe = '- firmware-brcm80211/%s' % backports_suite + wireless_firmware = 'firmware-brcm80211/%s' % backports_suite fix_firmware = False # CMA fixup: @@ -141,8 +141,8 @@ with open('raspi_master.yaml', 'r') as in_file: out_text = in_text \ .replace('__RELEASE__', suite) \ .replace('__SECURITY_SUITE__', security_suite) \ - .replace('__FIRMWARE_PKG__', raspi_firmware) \ - .replace('__WIRELESS_FIRMWARE_PKG_MAYBE__', wireless_firmware_maybe) \ + .replace('__RASPI_FIRMWARE__', raspi_firmware) \ + .replace('__WIRELESS_FIRMWARE__', wireless_firmware) \ .replace('__SERIAL_CONSOLE__', serial) \ .replace('__LINUX_IMAGE__', linux) \ .replace('__DTB__', dtb) \ @@ -155,6 +155,8 @@ with open('raspi_master.yaml', 'r') as in_file: out_text = align_replace(out_text, '__BACKPORTS__', backports_stanza.splitlines()) # Try not to keep lines where the placeholder was replaced - # with nothing at all: - filtered = [x for x in out_text.splitlines() if not re.match(r'^\s+$', x)] + # with nothing at all (including on a "list item" line): + filtered = [x for x in out_text.splitlines() + if not re.match(r'^\s+$', x) + and not re.match(r'^\s+-\s*$', x)] out_file.write('\n'.join(filtered) + "\n") diff --git a/raspi_master.yaml b/raspi_master.yaml index 771ed8e..78f6da7 100644 --- a/raspi_master.yaml +++ b/raspi_master.yaml @@ -79,9 +79,9 @@ steps: - parted - ssh - wpasupplicant - - __FIRMWARE_PKG__ - __LINUX_IMAGE__ - __WIRELESS_FIRMWARE_PKG_MAYBE__ + - __RASPI_FIRMWARE__ + - __WIRELESS_FIRMWARE__ tag: / unless: rootfs_unpacked From b339942ca42346d83175270c2475e12569e422a6 Mon Sep 17 00:00:00 2001 From: Cyril Brulebois Date: Sat, 28 Aug 2021 05:25:38 +0200 Subject: [PATCH 03/12] Fix missing trailing newline in sources.list Re-implement MR #49 by Diederik de Haas. --- raspi_master.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/raspi_master.yaml b/raspi_master.yaml index 78f6da7..9b9b930 100644 --- a/raspi_master.yaml +++ b/raspi_master.yaml @@ -49,11 +49,11 @@ steps: unless: rootfs_unpacked - create-file: /etc/apt/sources.list - trailing-newline: '1' - contents: | + contents: |+ deb http://deb.debian.org/debian __RELEASE__ main contrib non-free deb http://security.debian.org/debian-security __SECURITY_SUITE__ main contrib non-free __BACKPORTS__ + unless: rootfs_unpacked - copy-file: /etc/initramfs-tools/hooks/rpi-resizerootfs From 2108ed0a80315a9699c96d4793897997bde4adde Mon Sep 17 00:00:00 2001 From: Cyril Brulebois Date: Sat, 28 Aug 2021 05:30:30 +0200 Subject: [PATCH 04/12] Reorder replacements. Let's have variables being set and replacements being performed in roughly the same order. --- generate-recipe.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/generate-recipe.py b/generate-recipe.py index 7e2caac..17d2239 100755 --- a/generate-recipe.py +++ b/generate-recipe.py @@ -55,12 +55,6 @@ if version != '2': else: wireless_firmware = '' -# Serial console: -if version in ['1', '2']: - serial = 'ttyAMA0,115200' -elif version in ['3', '4']: - serial = 'ttyS1,115200' - # Pi 4 on buster requires some backports: backports_enable = False backports_suite = '%s-backports' % suite @@ -71,6 +65,12 @@ if version == '4' and suite == 'buster': wireless_firmware = 'firmware-brcm80211/%s' % backports_suite fix_firmware = False +# Serial console: +if version in ['1', '2']: + serial = 'ttyAMA0,115200' +elif version in ['3', '4']: + serial = 'ttyS1,115200' + # CMA fixup: extra_chroot_shell_cmds = [] if version == '4': @@ -140,14 +140,14 @@ with open('raspi_master.yaml', 'r') as in_file: in_text = in_file.read() out_text = in_text \ .replace('__RELEASE__', suite) \ + .replace('__ARCH__', arch) \ + .replace('__LINUX_IMAGE__', linux) \ + .replace('__DTB__', dtb) \ .replace('__SECURITY_SUITE__', security_suite) \ .replace('__RASPI_FIRMWARE__', raspi_firmware) \ .replace('__WIRELESS_FIRMWARE__', wireless_firmware) \ .replace('__SERIAL_CONSOLE__', serial) \ - .replace('__LINUX_IMAGE__', linux) \ - .replace('__DTB__', dtb) \ - .replace('__HOST__', hostname) \ - .replace('__ARCH__', arch) + .replace('__HOST__', hostname) out_text = align_replace(out_text, '__FIX_FIRMWARE_PKG_NAME__', fix_firmware_cmds) out_text = align_replace(out_text, '__EXTRA_ROOT_SHELL_CMDS__', extra_root_shell_cmds) From 1410839911c2ba2fe3f9d7c0075163e35d69ca1a Mon Sep 17 00:00:00 2001 From: Cyril Brulebois Date: Sat, 28 Aug 2021 05:34:27 +0200 Subject: [PATCH 05/12] Add an empty line before __EXTRA_CHROOT_SHELL_CMDS__ This means the generated recipes are getting two empty lines if there are no such commands (that's the case for everyone right now), but this emphasizes the existence of this placeholder, the same way as for its __EXTRA_ROOT_SHELL_CMDS__ twin. --- raspi_master.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/raspi_master.yaml b/raspi_master.yaml index 9b9b930..daee86f 100644 --- a/raspi_master.yaml +++ b/raspi_master.yaml @@ -142,6 +142,7 @@ steps: shell: | sed -i 's/^/console=__SERIAL_CONSOLE__ /' /boot/firmware/cmdline.txt sed -i 's/.dev.mmcblk0p2/LABEL=RASPIROOT/' /boot/firmware/cmdline.txt + __EXTRA_CHROOT_SHELL_CMDS__ # TODO(https://github.com/larswirzenius/vmdb2/issues/24): remove once vmdb From 0f8884c39cb83cf0e59957491ff5308cd0403a8c Mon Sep 17 00:00:00 2001 From: Cyril Brulebois Date: Sat, 28 Aug 2021 05:51:05 +0200 Subject: [PATCH 06/12] Replace the sed maze with one generate-recipe call for each combination. --- Makefile | 102 ++++--------------------------------------------------- 1 file changed, 7 insertions(+), 95 deletions(-) diff --git a/Makefile b/Makefile index d41c60c..9a22679 100644 --- a/Makefile +++ b/Makefile @@ -24,101 +24,13 @@ endif target_platforms: @echo $(platforms) -raspi_base_buster.yaml: raspi_master.yaml - cat raspi_master.yaml | \ - sed "s/__FIRMWARE_PKG__/raspi3-firmware/" | \ - sed "s/__RELEASE__/buster/" |\ - sed "s/__SECURITY_SUITE__/buster\/updates/" |\ - sed "s/__FIX_FIRMWARE_PKG_NAME__/sed -i s\/raspi-firmware\/raspi3-firmware\/ \$${ROOT?}\/etc\/systemd\/system\/rpi-reconfigure-raspi-firmware.service/" |\ - grep -v '__EXTRA_SHELL_CMDS__' > $@ - -raspi_1_buster.yaml: raspi_base_buster.yaml - cat raspi_base_buster.yaml | sed "s/__ARCH__/armel/" | \ - sed "s/__LINUX_IMAGE__/linux-image-rpi/" | \ - sed "s/__EXTRA_PKGS__/- firmware-brcm80211/" | \ - sed "s/__DTB__/\\/usr\\/lib\\/linux-image-*-rpi\\/bcm*rpi-*.dtb/" |\ - sed "s/__SERIAL_CONSOLE__/ttyAMA0,115200/" |\ - grep -v "__OTHER_APT_ENABLE__" |\ - sed "s/__HOST__/rpi1/" |\ - grep -v '__EXTRA_SHELL_CMDS__' > $@ - -raspi_2_buster.yaml: raspi_base_buster.yaml - cat raspi_base_buster.yaml | sed "s/__ARCH__/armhf/" | \ - sed "s/__LINUX_IMAGE__/linux-image-armmp/" | \ - grep -v "__EXTRA_PKGS__" | \ - sed "s/__DTB__/\\/usr\\/lib\\/linux-image-*-armmp\\/bcm*rpi*.dtb/" |\ - sed "s/__SERIAL_CONSOLE__/ttyAMA0,115200/" |\ - sed "s/__OTHER_APT_ENABLE__//" |\ - sed "s/__HOST__/rpi2/" |\ - grep -v '__EXTRA_SHELL_CMDS__' > $@ - -raspi_3_buster.yaml: raspi_base_buster.yaml - cat raspi_base_buster.yaml | sed "s/__ARCH__/arm64/" | \ - sed "s/__LINUX_IMAGE__/linux-image-arm64/" | \ - sed "s/__EXTRA_PKGS__/- firmware-brcm80211/" | \ - sed "s/__DTB__/\\/usr\\/lib\\/linux-image-*-arm64\\/broadcom\\/bcm*rpi*.dtb/" |\ - sed "s/__SERIAL_CONSOLE__/ttyS1,115200/" |\ - sed "s/__OTHER_APT_ENABLE__//" |\ - sed "s/__HOST__/rpi3/" |\ - grep -v '__EXTRA_SHELL_CMDS__' > $@ - -raspi_4_buster.yaml: raspi_base_buster.yaml - cat raspi_base_buster.yaml | sed "s/__ARCH__/arm64/" | \ - sed "s#raspi3-firmware#raspi-firmware/buster-backports#" | \ - sed "s#apt-get update#echo 'APT::Default-Release \"buster\";' > /etc/apt/apt.conf\n apt-get update#" | \ - sed "s#\(RASPIROOT.*cmdline.txt\)#\1\n sed -i 's/cma=64M //' /boot/firmware/cmdline.txt\n sed -i 's/cma=\\\$$CMA //' /etc/kernel/postinst.d/z50-raspi-firmware#" | \ - sed "s/__LINUX_IMAGE__/linux-image-arm64\/buster-backports/" | \ - sed "s/__EXTRA_PKGS__/- firmware-brcm80211\/buster-backports/" | \ - sed "s/__DTB__/\\/usr\\/lib\\/linux-image-*-arm64\\/broadcom\\/bcm*rpi*.dtb/" |\ - sed "s/__SERIAL_CONSOLE__/ttyS1,115200/" |\ - sed "s/__OTHER_APT_ENABLE__/deb http:\/\/deb.debian.org\/debian\/ buster-backports main contrib non-free # raspi 4 needs a kernel and raspi-firmware newer than buster's/" |\ - sed "s/__HOST__/rpi4/" |\ - grep -v '__EXTRA_SHELL_CMDS__' > $@ - -raspi_base_bullseye.yaml: raspi_master.yaml - cat raspi_master.yaml | \ - sed "s/__RELEASE__/bullseye/" |\ - sed "s/__FIRMWARE_PKG__/raspi-firmware/" | \ - grep -v "__OTHER_APT_ENABLE__" |\ - grep -v "__FIX_FIRMWARE_PKG_NAME__" |\ - sed "s/__SECURITY_SUITE__/bullseye-security/" > $@ - -raspi_1_bullseye.yaml: raspi_base_bullseye.yaml - cat raspi_base_bullseye.yaml | sed "s/__ARCH__/armel/" | \ - sed "s/__LINUX_IMAGE__/linux-image-rpi/" | \ - sed "s/__EXTRA_PKGS__/- firmware-brcm80211/" | \ - sed "s/__DTB__/\\/usr\\/lib\\/linux-image-*-rpi\\/bcm*rpi-*.dtb/" |\ - sed "s/__SERIAL_CONSOLE__/ttyAMA0,115200/" |\ - sed "s/__HOST__/rpi_1/" |\ - grep -v '__EXTRA_SHELL_CMDS__' > $@ - -raspi_2_bullseye.yaml: raspi_base_bullseye.yaml - cat raspi_base_bullseye.yaml | sed "s/__ARCH__/armhf/" | \ - sed "s/__LINUX_IMAGE__/linux-image-armmp/" | \ - grep -v "__EXTRA_PKGS__" | \ - sed "s/__DTB__/\\/usr\\/lib\\/linux-image-*-armmp\\/bcm*rpi*.dtb/" |\ - sed "s/__SERIAL_CONSOLE__/ttyAMA0,115200/" |\ - sed "s/__HOST__/rpi_2/" |\ - grep -v '__EXTRA_SHELL_CMDS__' > $@ - -raspi_3_bullseye.yaml: raspi_base_bullseye.yaml - cat raspi_base_bullseye.yaml | sed "s/__ARCH__/arm64/" | \ - sed "s/__LINUX_IMAGE__/linux-image-arm64/" | \ - sed "s/__EXTRA_PKGS__/- firmware-brcm80211/" | \ - sed "s/__DTB__/\\/usr\\/lib\\/linux-image-*-arm64\\/broadcom\\/bcm*rpi*.dtb/" |\ - sed "s/__SERIAL_CONSOLE__/ttyS1,115200/" |\ - sed "s/__HOST__/rpi_3/" |\ - grep -v '__EXTRA_SHELL_CMDS__' > $@ - -raspi_4_bullseye.yaml: raspi_base_bullseye.yaml - cat raspi_base_bullseye.yaml | sed "s/__ARCH__/arm64/" | \ - sed "s#\(RASPIROOT.*cmdline.txt\)#\1\n sed -i 's/cma=64M //' /boot/firmware/cmdline.txt\n sed -i 's/cma=\\\$$CMA //' /etc/kernel/postinst.d/z50-raspi-firmware#" | \ - sed "s/__LINUX_IMAGE__/linux-image-arm64/" | \ - sed "s/__EXTRA_PKGS__/- firmware-brcm80211/" | \ - sed "s/__DTB__/\\/usr\\/lib\\/linux-image-*-arm64\\/broadcom\\/bcm*rpi*.dtb/" |\ - sed "s/__SERIAL_CONSOLE__/ttyS1,115200/" |\ - sed "s/__HOST__/rpi_4/" |\ - grep -v '__EXTRA_SHELL_CMDS__' > $@ +# Generate targets based on all family * release combinations: +define dynamic_yaml_target = + raspi_$(1)_$(2).yaml: raspi_master.yaml generate-recipe.py + raspi_$(1)_$(2).yaml: + ./generate-recipe.py $(1) $(2) +endef +$(foreach release,$(BUILD_RELEASES),$(foreach family,$(BUILD_FAMILIES),$(eval $(call dynamic_yaml_target,$(family),$(release))))) %.img.sha256: %.img echo $@ From 252a6948489e8c15ef1cd9e714dc98dd427b16dc Mon Sep 17 00:00:00 2001 From: Cyril Brulebois Date: Sat, 28 Aug 2021 05:52:51 +0200 Subject: [PATCH 07/12] Improve loop readability. Split dynamic target generation across several lines to make the nested loops more obvious. Backslashes are needed for make to be happy about what would otherwise be detected as unfinished foreach function calls. --- Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9a22679..eeb6f82 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,9 @@ define dynamic_yaml_target = raspi_$(1)_$(2).yaml: ./generate-recipe.py $(1) $(2) endef -$(foreach release,$(BUILD_RELEASES),$(foreach family,$(BUILD_FAMILIES),$(eval $(call dynamic_yaml_target,$(family),$(release))))) +$(foreach release,$(BUILD_RELEASES), \ + $(foreach family,$(BUILD_FAMILIES), \ + $(eval $(call dynamic_yaml_target,$(family),$(release))))) %.img.sha256: %.img echo $@ From 5f7a28d8ec25a68fe86540fcd500166b123b7509 Mon Sep 17 00:00:00 2001 From: Cyril Brulebois Date: Sat, 20 Nov 2021 03:56:59 +0100 Subject: [PATCH 08/12] Add support for bookworm. --- Makefile | 2 +- generate-recipe.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index eeb6f82..f7f3f28 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ all: shasums # List all the supported and built Pi platforms here. They get expanded # to names like 'raspi_2_buster.yaml' and 'raspi_3_bullseye.img.xz'. BUILD_FAMILIES := 1 2 3 4 -BUILD_RELEASES := buster bullseye +BUILD_RELEASES := buster bullseye bookworm platforms := $(foreach plat, $(BUILD_FAMILIES),$(foreach rel, $(BUILD_RELEASES), raspi_$(plat)_$(rel))) diff --git a/generate-recipe.py b/generate-recipe.py index 17d2239..2993138 100755 --- a/generate-recipe.py +++ b/generate-recipe.py @@ -17,7 +17,7 @@ if version not in ["1", "2", "3", "4"]: sys.exit(1) suite = sys.argv[2] -if suite not in ['buster', 'bullseye']: +if suite not in ['buster', 'bullseye', 'bookworm']: print("E: unsupported suite %s" % suite, file=sys.stderr) sys.exit(1) target_yaml = 'raspi_%s_%s.yaml' % (version, suite) From 592c0df22e3edf4ff416946473b1862a5c1627b9 Mon Sep 17 00:00:00 2001 From: Cyril Brulebois Date: Sat, 20 Nov 2021 04:09:52 +0100 Subject: [PATCH 09/12] Adjust /etc/machine-id logic for buster. Commit 26a7de63b0bb3de1b5d0c4d0529240721c322dbb in master: /etc/machine-id needs to exist and be empty on buster, while bullseye needs this file not to exist at all. For now, treat both bullseye and bookworm the same way. --- generate-recipe.py | 9 ++++++++- raspi_master.yaml | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/generate-recipe.py b/generate-recipe.py index 2993138..dbfe345 100755 --- a/generate-recipe.py +++ b/generate-recipe.py @@ -113,6 +113,12 @@ else: # deb http://deb.debian.org/debian %s main contrib non-free """ % backports_suite +# Buster requires an existing, empty /etc/machine-id file: +if suite == 'buster': + touch_machine_id = 'touch /etc/machine-id' +else: + touch_machine_id = '' + ### Write results: @@ -147,7 +153,8 @@ with open('raspi_master.yaml', 'r') as in_file: .replace('__RASPI_FIRMWARE__', raspi_firmware) \ .replace('__WIRELESS_FIRMWARE__', wireless_firmware) \ .replace('__SERIAL_CONSOLE__', serial) \ - .replace('__HOST__', hostname) + .replace('__HOST__', hostname) \ + .replace('__TOUCH_MACHINE_ID__', touch_machine_id) out_text = align_replace(out_text, '__FIX_FIRMWARE_PKG_NAME__', fix_firmware_cmds) out_text = align_replace(out_text, '__EXTRA_ROOT_SHELL_CMDS__', extra_root_shell_cmds) diff --git a/raspi_master.yaml b/raspi_master.yaml index daee86f..bfae52a 100644 --- a/raspi_master.yaml +++ b/raspi_master.yaml @@ -163,3 +163,4 @@ steps: - chroot: / shell: | rm -f /etc/machine-id /var/lib/dbus/machine-id + __TOUCH_MACHINE_ID__ From 650da9d35122b745877e6ce3892c9699d730ef92 Mon Sep 17 00:00:00 2001 From: Cyril Brulebois Date: Sat, 20 Nov 2021 04:16:13 +0100 Subject: [PATCH 10/12] Pull firmware-brcm80211 from buster-backports for the Pi 3. Commit 611b5c6d5cd0ac0b0615f3300beb2415b2f99fe0 in master. --- generate-recipe.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/generate-recipe.py b/generate-recipe.py index dbfe345..26c8e9b 100755 --- a/generate-recipe.py +++ b/generate-recipe.py @@ -65,6 +65,10 @@ if version == '4' and suite == 'buster': wireless_firmware = 'firmware-brcm80211/%s' % backports_suite fix_firmware = False +if version == '3' and suite == 'buster': + backports_enable = "# raspi 3 needs firmware-brcm80211 newer than buster's for wifi" + wireless_firmware = 'firmware-brcm80211/%s' % backports_suite + # Serial console: if version in ['1', '2']: serial = 'ttyAMA0,115200' From 66f43b854f43f72018c2d8168f71bc9f6bb95f8c Mon Sep 17 00:00:00 2001 From: Diederik de Haas Date: Thu, 9 Sep 2021 01:00:59 +0200 Subject: [PATCH 11/12] Fix _clean_shasums to also remove *.img.sha256. Commit 422a0d60 fixed the img.sha256 target itself, but it didn't update the corresponding clean variant, so add that too. Signed-off-by: Diederik de Haas (cherry picked from commit 60513d46f6f687a2582be3b9a9589ea6259813b9) Signed-off-by: Cyril Brulebois --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f7f3f28..408d6a4 100644 --- a/Makefile +++ b/Makefile @@ -65,7 +65,7 @@ _clean_xzimages: _clean_bmaps: rm -f $(addsuffix .img.bmap,$(platforms)) _clean_shasums: - rm -f $(addsuffix .sha256,$(platforms)) $(addsuffix .img.xz.sha256,$(platforms)) + rm -f $(addsuffix .img.sha256,$(platforms)) $(addsuffix .img.xz.sha256,$(platforms)) _clean_logs: rm -f $(addsuffix .log,$(platforms)) _clean_tarballs: From cab748ab0c63913dbef44e33dece9df48c52b7e2 Mon Sep 17 00:00:00 2001 From: Cyril Brulebois Date: Sat, 20 Nov 2021 15:10:20 +0100 Subject: [PATCH 12/12] Drop cma= tweaking in /etc/kernel/postinst.d/z50-raspi-firmware. See 3f9e671fedd54b4a332e5a93e156489600203fad in the boot-consistency branch, later adjusted in 2b2bb9d6d78404c53bc1ccd31e8094aa1acb2921 for the master branch and the new bookworm builds. With the pythonize approach, a single change is needed. --- generate-recipe.py | 1 - 1 file changed, 1 deletion(-) diff --git a/generate-recipe.py b/generate-recipe.py index 26c8e9b..a8f428f 100755 --- a/generate-recipe.py +++ b/generate-recipe.py @@ -80,7 +80,6 @@ extra_chroot_shell_cmds = [] if version == '4': extra_chroot_shell_cmds = [ "sed -i 's/cma=64M //' /boot/firmware/cmdline.txt", - "sed -i 's/cma=$CMA //' /etc/kernel/postinst.d/z50-raspi-firmware", ] # XXX: The disparity between suite seems to be a bug, pick a naming