Generate recipes using a Python script instead of multiplying sed calls.

This is a proof of concept rather than an ideal, final situation.

It can be used this way:

    for v in 1 2 3 4 ; do
      for s in buster bullseye; do
        ./generate-recipe.py $v $s
      done
    done

and it has been verified to produce very similar results compared to the
existing many-sed approach.

Differences are as follows:
 - Missing newline after some backports stanza, due to the removal of
   the other APT line. There's already MR#51 that aims at fixing some
   newline-related issues anyway, so this can be addressed separately.
 - Less schizophrenia in the generated sources.list for buster/4, as we
   are now only showing a reason for enabling the backports, instead
   of starting by explaining why backports are disabled by default.
 - Dropping APT::Default-Release = buster in the buster/4 case, which
   is no longer needed as we are pulling things from buster-backports
   rather than pulling them from unstable (see 57e90df103).
 - No longer trying to fix the firmware package name by throwing a
   broken sed at rpi-reconfigure-raspi-firmware.service in the buster/4
   case: the syntax was buggy and fixing it would have made us try to
   replace raspi-firmware with raspi-firmware/buster-backports, while
   the correct thing to do is to not touch it in the first place
   (raspi-firmware is the correct name for the firmware package, pulled
   from buster-backports).

As a side effect, this transforms the existing __EXTRA_SHELL_CMDS__ into
a slightly more explicit __EXTRA_ROOT_SHELL_CMDS__ which now has its
__EXTRA_CHROOT_SHELL_CMDS__ twin. That's the entry point that was
missing and made 45cb5619d4 necessary in the past.

generate-recipe.py

@@ -0,0 +1,160 @@
+#!/usr/bin/python3
+
+import re
+import sys
+
+# pylint: disable=invalid-name
+
+### Sanity/usage checks
+
+if len(sys.argv) != 3:
+    print("E: need 2 arguments", file=sys.stderr)
+    sys.exit(1)
+
+version = sys.argv[1]
+if version not in ["1", "2", "3", "4"]:
+    print("E: unsupported version %s" % version, file=sys.stderr)
+    sys.exit(1)
+
+suite = sys.argv[2]
+if suite not in ['buster', 'bullseye']:
+    print("E: unsupported suite %s" % suite, file=sys.stderr)
+    sys.exit(1)
+target_yaml = 'raspi_%s_%s.yaml' % (version, suite)
+
+
+### Setting variables based on suite and version starts here
+
+# Arch, kernel, DTB:
+if version == '1':
+    arch = 'armel'
+    linux = 'linux-image-rpi'
+    dtb = '/usr/lib/linux-image-*-rpi/bcm*rpi-*.dtb'
+elif version == '2':
+    arch = 'armhf'
+    linux = 'linux-image-armmp'
+    dtb = '/usr/lib/linux-image-*-armmp/bcm*rpi*.dtb'
+elif version in ['3', '4']:
+    arch = 'arm64'
+    linux = 'linux-image-arm64'
+    dtb = '/usr/lib/linux-image-*-arm64/broadcom/bcm*rpi*.dtb'
+
+# APT and default firmware (name + handling)
+if suite == 'buster':
+    security_suite = '%s/updates' % suite
+    raspi_firmware = 'raspi3-firmware'
+    fix_firmware = True
+else:
+    security_suite = '%s-security' % suite
+    raspi_firmware = 'raspi-firmware'
+    fix_firmware = False
+
+# Extra wireless firmware:
+if version != '2':
+    wireless_firmware_maybe = '- firmware-brcm80211'
+else:
+    wireless_firmware_maybe = ''
+
+# Serial console:
+if version in ['1', '2']:
+    serial = 'ttyAMA0,115200'
+elif version in ['3', '4']:
+    serial = 'ttyS1,115200'
+
+# Pi 4 on buster requires some backports:
+backports_enable = False
+backports_suite = '%s-backports' % suite
+if version == '4' and suite == 'buster':
+    backports_enable = "# raspi 4 needs kernel and firmware newer than buster's"
+    linux = '%s/%s' % (linux, backports_suite)
+    raspi_firmware = 'raspi-firmware/%s' % backports_suite
+    wireless_firmware_maybe = '- firmware-brcm80211/%s' % backports_suite
+    fix_firmware = False
+
+# CMA fixup:
+extra_chroot_shell_cmds = []
+if version == '4':
+    extra_chroot_shell_cmds = [
+        "sed -i 's/cma=64M //' /boot/firmware/cmdline.txt",
+        "sed -i 's/cma=$CMA //' /etc/kernel/postinst.d/z50-raspi-firmware",
+    ]
+
+# XXX: The disparity between suite seems to be a bug, pick a naming
+# and stick to it!
+#
+# Hostname:
+if suite == 'buster':
+    hostname = 'rpi%s' % version
+else:
+    hostname = 'rpi_%s' % version
+
+# Nothing yet!
+extra_root_shell_cmds = []
+
+
+### The following prepares substitutions based on variables set earlier
+
+# Commands to fix the firmware name in the systemd unit:
+if fix_firmware:
+    fix_firmware_cmds = ['sed -i s/raspi-firmware/raspi3-firmware/ ${ROOT?}/etc/systemd/system/rpi-reconfigure-raspi-firmware.service']
+else:
+    fix_firmware_cmds = []
+
+# Enable backports with a reason, or add commented-out entry:
+if backports_enable:
+    backports_stanza = """
+%s
+deb http://deb.debian.org/debian/ %s main contrib non-free
+""" % (backports_enable, backports_suite)
+else:
+    backports_stanza = """
+# Backports are _not_ enabled by default.
+# Enable them by uncommenting the following line:
+# deb http://deb.debian.org/debian %s main contrib non-free
+""" % backports_suite
+
+
+### Write results:
+
+def align_replace(text, pattern, replacement):
+    """
+    This helper lets us keep the indentation of the matched pattern
+    with the upcoming replacement, across multiple lines. Naive
+    implementation, please make it more pythonic!
+    """
+    lines = text.splitlines()
+    for i, line in enumerate(lines):
+        m = re.match(r'^(\s+)%s' % pattern, line)
+        if m:
+            indent = m.group(1)
+            del lines[i]
+            for r in replacement:
+                lines.insert(i, '%s%s' % (indent, r))
+                i = i + 1
+            break
+    return '\n'. join(lines) + '\n'
+
+
+with open('raspi_master.yaml', 'r') as in_file:
+    with open(target_yaml, 'w') as out_file:
+        in_text = in_file.read()
+        out_text = in_text \
+            .replace('__RELEASE__', suite) \
+            .replace('__SECURITY_SUITE__', security_suite) \
+            .replace('__FIRMWARE_PKG__', raspi_firmware) \
+            .replace('__WIRELESS_FIRMWARE_PKG_MAYBE__', wireless_firmware_maybe) \
+            .replace('__SERIAL_CONSOLE__', serial) \
+            .replace('__LINUX_IMAGE__', linux) \
+            .replace('__DTB__', dtb) \
+            .replace('__HOST__', hostname) \
+            .replace('__ARCH__', arch)
+
+        out_text = align_replace(out_text, '__FIX_FIRMWARE_PKG_NAME__', fix_firmware_cmds)
+        out_text = align_replace(out_text, '__EXTRA_ROOT_SHELL_CMDS__', extra_root_shell_cmds)
+        out_text = align_replace(out_text, '__EXTRA_CHROOT_SHELL_CMDS__', extra_chroot_shell_cmds)
+        out_text = align_replace(out_text, '__BACKPORTS__', backports_stanza.splitlines())
+
+        # Try not to keep lines where the placeholder was replaced
+        # with nothing at all:
+        filtered = [x for x in out_text.splitlines() if not re.match(r'^\s+$', x)]
+        out_file.write('\n'.join(filtered) + "\n")

raspi_master.yaml

@@ -53,10 +53,7 @@ steps:
     contents: |
       deb http://deb.debian.org/debian __RELEASE__ main contrib non-free
       deb http://security.debian.org/debian-security __SECURITY_SUITE__ main contrib non-free
-      # Backports are _not_ enabled by default.
+      __BACKPORTS__
-      # Enable them by uncommenting the following line:
-      # deb http://deb.debian.org/debian __RELEASE__-backports main contrib non-free
-      __OTHER_APT_ENABLE__
     unless: rootfs_unpacked
 
   - copy-file: /etc/initramfs-tools/hooks/rpi-resizerootfs
@@ -84,7 +81,7 @@ steps:
     - wpasupplicant
     - __FIRMWARE_PKG__
     - __LINUX_IMAGE__
-    __EXTRA_PKGS__
+    __WIRELESS_FIRMWARE_PKG_MAYBE__
     tag: /
     unless: rootfs_unpacked
 
@@ -121,7 +118,7 @@ steps:
       ln -s /etc/systemd/system/rpi-generate-ssh-host-keys.service "${ROOT?}/etc/systemd/system/multi-user.target.requires/rpi-generate-ssh-host-keys.service"
       rm -f "${ROOT?}"/etc/ssh/ssh_host_*_key*
 
-      __EXTRA_SHELL_CMDS__
+      __EXTRA_ROOT_SHELL_CMDS__
     root-fs: /
 
   # Copy the relevant device tree files to the boot partition
@@ -145,6 +142,7 @@ steps:
     shell: |
       sed -i 's/^/console=__SERIAL_CONSOLE__ /' /boot/firmware/cmdline.txt
       sed -i 's/.dev.mmcblk0p2/LABEL=RASPIROOT/' /boot/firmware/cmdline.txt
+      __EXTRA_CHROOT_SHELL_CMDS__
 
   # TODO(https://github.com/larswirzenius/vmdb2/issues/24): remove once vmdb
   # clears /etc/resolv.conf on its own.