Hibbian image spec for pi4 added

Support for the former was removed in vmdb2 0.27, and unstable/testing
have 0.28 now. The switch is known to work in bookworm as well (0.26).

It's supposed to be transparent, but due to different options in both
plugins, one has to specify require_empty_target for the debootstrap
plugin (which wasn't needed for qemu-debootstrap).

.gitignore

@@ -20,3 +20,7 @@ raspi_1_trixie.yaml
 raspi_2_trixie.yaml
 raspi_3_trixie.yaml
 raspi_4_trixie.yaml
+*.log
+*.img.xz
+*.img.gz
+*.img

README.md

@@ -33,10 +33,11 @@ Debian Bullseye (11) or higher system:
 * time
 * vmdb2 (>= 0.17)
 * python3
+* zerofree (because of [#1021341](https://bugs.debian.org/1021341))
 
 To install these (as root):
 ```shell
-   apt install -y vmdb2 dosfstools qemu-utils qemu-user-static debootstrap binfmt-support time kpartx bmap-tools python3
+   apt install -y vmdb2 dosfstools qemu-utils qemu-user-static debootstrap binfmt-support time kpartx bmap-tools python3 zerofree
    apt install -y fakemachine
 ```
 
@@ -116,7 +117,7 @@ Alternatively, if you don't have `bmap-tools` installed, you can use
 `dd` with the compressed image:
 
 ```shell
-xzcat raspi_3_bullseye.img | dd of=/dev/mmcblk0 bs=64k oflag=dsync status=progress
+xzcat raspi_3_bullseye.img.xz | dd of=/dev/mmcblk0 bs=64k oflag=dsync status=progress
 ```
 
 Or with the uncompressed image:

generate-recipe.py

@@ -40,10 +40,6 @@ elif version in ['3', '4']:
     linux = 'linux-image-arm64'
     dtb = '/usr/lib/linux-image-*-arm64/broadcom/bcm*rpi*.dtb'
 
-# APT and default firmware (name + handling)
-raspi_firmware = 'raspi-firmware'
-fix_firmware = False
-
 # Bookworm introduced the 'non-free-firmware' component¹; before that,
 # raspi-firmware was in 'non-free'
 #
@@ -67,7 +63,8 @@ if version != '2':
 else:
     bluetooth_firmware = ''
 
-# Pi 4 on buster requires some backports:
+# Pi 4 on buster required some backports. Let's keep variables around, ready to
+# be used whenever we need to pull specific things from backports.
 backports_enable = False
 backports_suite = '%s-backports' % suite
 
@@ -84,9 +81,6 @@ if version == '4':
         "sed -i 's/cma=64M //' /boot/firmware/cmdline.txt",
     ]
 
-# XXX: The disparity between suite seems to be a bug, pick a naming
-# and stick to it!
-#
 # Hostname:
 hostname = 'rpi_%s' % version
 
@@ -96,12 +90,6 @@ extra_root_shell_cmds = []
 
 ### The following prepares substitutions based on variables set earlier
 
-# Commands to fix the firmware name in the systemd unit:
-if fix_firmware:
-    fix_firmware_cmds = ['sed -i s/raspi-firmware/raspi3-firmware/ ${ROOT?}/etc/systemd/system/rpi-reconfigure-raspi-firmware.service']
-else:
-    fix_firmware_cmds = []
-
 # Enable backports with a reason, or add commented-out entry:
 if backports_enable:
     backports_stanza = """
@@ -116,12 +104,6 @@ else:
 # deb http://deb.debian.org/debian %s main %s
 """ % (backports_suite, firmware_component)
 
-# Buster requires an existing, empty /etc/machine-id file:
-touch_machine_id = 'echo "uninitialized" > /etc/machine-id'
-
-# Buster shipped timesyncd directly into systemd:
-systemd_timesyncd = 'systemd-timesyncd'
-
 gitcommit = subprocess.getoutput("git show -s --pretty='format:%C(auto)%h (%s, %ad)' --date=short ")
 buildtime = subprocess.getoutput("date --utc +'%Y-%m-%d %H:%M'")
 
@@ -156,17 +138,13 @@ with open('raspi_master.yaml', 'r') as in_file:
             .replace('__FIRMWARE_COMPONENT_OLD__', firmware_component_old) \
             .replace('__LINUX_IMAGE__', linux) \
             .replace('__DTB__', dtb) \
-            .replace('__SYSTEMD_TIMESYNCD__', systemd_timesyncd) \
-            .replace('__RASPI_FIRMWARE__', raspi_firmware) \
             .replace('__WIRELESS_FIRMWARE__', wireless_firmware) \
             .replace('__BLUETOOTH_FIRMWARE__', bluetooth_firmware) \
             .replace('__SERIAL_CONSOLE__', serial) \
             .replace('__HOST__', hostname) \
-            .replace('__TOUCH_MACHINE_ID__', touch_machine_id) \
             .replace('__GITCOMMIT__', gitcommit) \
             .replace('__BUILDTIME__', buildtime)
 
-        out_text = align_replace(out_text, '__FIX_FIRMWARE_PKG_NAME__', fix_firmware_cmds)
         out_text = align_replace(out_text, '__EXTRA_ROOT_SHELL_CMDS__', extra_root_shell_cmds)
         out_text = align_replace(out_text, '__EXTRA_CHROOT_SHELL_CMDS__', extra_chroot_shell_cmds)
         out_text = align_replace(out_text, '__BACKPORTS__', backports_stanza.splitlines())

raspi_4_hibbian.yaml

@@ -0,0 +1,199 @@
+---
+# See https://wiki.debian.org/RaspberryPi3 for known issues and more details.
+# image.yml based on revision: ff7fdbf (Switch from qemu-debootstrap to debootstrap., 2024-01-01)
+
+steps:
+  - mkimg: "{{ output }}"
+    size: 2500M
+
+  - mklabel: msdos
+    device: "{{ output }}"
+
+  - mkpart: primary
+    fs-type: 'fat32'
+    device: "{{ output }}"
+    start: 4MiB
+    end: 512MiB
+    tag: tag-firmware
+
+  - mkpart: primary
+    device: "{{ output }}"
+    start: 512MiB
+    end: 100%
+    tag: tag-root
+
+  - kpartx: "{{ output }}"
+
+  - mkfs: vfat
+    partition: tag-firmware
+    label: RASPIFIRM
+
+  - mkfs: ext4
+    partition: tag-root
+    label: RASPIROOT
+
+  - mount: tag-root
+
+  - mount: tag-firmware
+    mount-on: tag-root
+    dirname: '/boot/firmware'
+
+  - unpack-rootfs: tag-root
+
+  - debootstrap: bookworm
+    require_empty_target: false
+    mirror: http://deb.debian.org/debian
+    target: tag-root
+    arch: arm64
+    components:
+      - main
+      - non-free-firmware
+      - non-free
+    unless: rootfs_unpacked
+
+  - create-file: /etc/apt/sources.list
+    contents: |+
+      deb http://deb.debian.org/debian bookworm main non-free-firmware non-free
+      deb http://deb.debian.org/debian bookworm-updates main non-free-firmware non-free
+      deb http://security.debian.org/debian-security bookworm-security main non-free-firmware non-free
+      # Backports are _not_ enabled by default.
+      # Enable them by uncommenting the following line:
+      deb http://deb.debian.org/debian bookworm-backports main non-free-firmware
+
+  - create-file: /etc/apt/preferences.d/hibbian.pref
+    contents: |+
+      Package: *
+      Pin: release o=Hibbian
+      Pin-Priority: 600
+
+  - create-file: /etc/apt/preferences.d/hibbian.pref
+    contents: |+
+      Package: linux-image
+      Pin: release o=Debian Backports
+      Pin-Priority: 500
+
+  - create-file: /etc/apt/sources.list.d/hibbian.list
+    contents: |+
+      deb http://repo.hibbian.org/hibbian bookworm-hibbian-unstable main non-free-firmware non-free
+
+  - copy-file: /etc/apt/trusted.gpg.d/hibbian-archive-keyring.gpg
+    src: etc/apt/trusted.gpg.d/hibbian-archive-keyring.gpg
+    perm: 0755
+
+  - copy-file: /etc/initramfs-tools/hooks/rpi-resizerootfs
+    src: rootfs/etc/initramfs-tools/hooks/rpi-resizerootfs
+    perm: 0755
+    unless: rootfs_unpacked
+
+  - copy-file: /etc/initramfs-tools/scripts/local-bottom/rpi-resizerootfs
+    src: rootfs/etc/initramfs-tools/scripts/local-bottom/rpi-resizerootfs
+    perm: 0755
+    unless: rootfs_unpacked
+
+  - apt: install
+    packages:
+      - ca-certificates
+      - dosfstools
+      - iw
+      - parted
+      - openssh-server
+      - network-manager
+      - systemd-timesyncd
+      - linux-image-arm64
+      - raspi-firmware
+      - firmware-brcm80211
+      - bluez-firmware
+      - base-files
+      - linbpq
+      - sudo
+      - vim-nox
+      - net-tools
+    tag: tag-root
+
+  - cache-rootfs: tag-root
+    unless: rootfs_unpacked
+
+  - shell: |
+      echo "hibbian-$(date +%Y%m%d)" > "${ROOT?}/etc/hostname"
+
+      # Allow root logins locally with no password
+      sed -i 's,root:[^:]*:,root::,' "${ROOT?}/etc/shadow"
+
+      install -m 644 -o root -g root rootfs/etc/fstab "${ROOT?}/etc/fstab"
+
+      install -m 644 -o root -g root rootfs/etc/network/interfaces.d/eth0 "${ROOT?}/etc/network/interfaces.d/eth0"
+      install -m 600 -o root -g root rootfs/etc/network/interfaces.d/wlan0 "${ROOT?}/etc/network/interfaces.d/wlan0"
+
+      install -m 755 -o root -g root rootfs/usr/local/sbin/rpi-set-sysconf "${ROOT?}/usr/local/sbin/rpi-set-sysconf"
+      install -m 644 -o root -g root rootfs/etc/systemd/system/rpi-set-sysconf.service "${ROOT?}/etc/systemd/system/"
+      install -m 644 -o root -g root rootfs/boot/firmware/sysconf.txt "${ROOT?}/boot/firmware/sysconf.txt"
+      mkdir -p "${ROOT?}/etc/systemd/system/basic.target.requires/"
+      ln -s /etc/systemd/system/rpi-set-sysconf.service "${ROOT?}/etc/systemd/system/basic.target.requires/rpi-set-sysconf.service"
+
+      # Resize script is now in the initrd for first boot; no need to ship it.
+      rm -f "${ROOT?}/etc/initramfs-tools/hooks/rpi-resizerootfs"
+      rm -f "${ROOT?}/etc/initramfs-tools/scripts/local-bottom/rpi-resizerootfs"
+
+      install -m 644 -o root -g root rootfs/etc/systemd/system/rpi-reconfigure-raspi-firmware.service "${ROOT?}/etc/systemd/system/"
+      mkdir -p "${ROOT?}/etc/systemd/system/multi-user.target.requires/"
+      ln -s /etc/systemd/system/rpi-reconfigure-raspi-firmware.service "${ROOT?}/etc/systemd/system/multi-user.target.requires/rpi-reconfigure-raspi-firmware.service"
+
+      install -m 644 -o root -g root rootfs/etc/systemd/system/rpi-generate-ssh-host-keys.service "${ROOT?}/etc/systemd/system/"
+      ln -s /etc/systemd/system/rpi-generate-ssh-host-keys.service "${ROOT?}/etc/systemd/system/multi-user.target.requires/rpi-generate-ssh-host-keys.service"
+      rm -f "${ROOT?}"/etc/ssh/ssh_host_*_key*
+
+    root-fs: tag-root
+
+  # Copy the relevant device tree files to the boot partition
+  - chroot: tag-root
+    shell: |
+      install -m 644 -o root -g root /usr/lib/linux-image-*-arm64/broadcom/bcm*rpi*.dtb /boot/firmware/
+
+  # Clean up archive cache (likely not useful) and lists (likely outdated) to
+  # reduce image size by several hundred megabytes.
+  - chroot: tag-root
+    shell: |
+      apt-get clean
+      rm -rf /var/lib/apt/lists
+
+  # Modify the kernel commandline we take from the firmware to boot from
+  # the partition labeled raspiroot instead of forcing it to mmcblk0p2.
+  # Also insert the serial console right before the root= parameter.
+  #
+  # These changes will be overwritten after the hardware is probed
+  # after dpkg reconfigures raspi-firmware (upon first boot), so make
+  # sure we don't lose label-based booting.
+  - chroot: tag-root
+    shell: |
+      sed -i 's/root=/console=ttyS1,115200 root=/' /boot/firmware/cmdline.txt
+      sed -i 's#root=/dev/mmcblk0p2#root=LABEL=RASPIROOT#' /boot/firmware/cmdline.txt
+      sed -i 's/^#ROOTPART=.*/ROOTPART=LABEL=RASPIROOT/' /etc/default/raspi*-firmware
+
+      sed -i 's/cma=64M //' /boot/firmware/cmdline.txt
+
+  # TODO(https://github.com/larswirzenius/vmdb2/issues/24): remove once vmdb
+  # clears /etc/resolv.conf on its own.
+  - shell: |
+      rm "${ROOT?}/etc/resolv.conf"
+    root-fs: tag-root
+
+  # Clear /etc/machine-id and /var/lib/dbus/machine-id, as both should
+  # be auto-generated upon first boot. From the manpage
+  # (machine-id(5)):
+  #
+  #   For normal operating system installations, where a custom image is
+  #   created for a specific machine, /etc/machine-id should be
+  #   populated during installation.
+  #
+  # Note this will also trigger ConditionFirstBoot=yes for systemd.
+  # On Buster, /etc/machine-id should be an emtpy file, not an absent file
+  # On Bullseye, /etc/machine-id should not exist in an image
+  - chroot: tag-root
+    shell: |
+      rm -f /etc/machine-id /var/lib/dbus/machine-id
+      echo "uninitialized" > /etc/machine-id
+
+  # Create /etc/raspi-image-id to know, from what commit the image was built
+  - chroot: tag-root
+    shell: |
+      echo "image based on revision: ff7fdbf (Switch from qemu-debootstrap to debootstrap., 2024-01-01) and built in 2024 with love from Hibby" > "/etc/raspi-image-id"

raspi_master.yaml

@@ -40,7 +40,8 @@ steps:
 
   - unpack-rootfs: tag-root
 
-  - qemu-debootstrap: __RELEASE__
+  - debootstrap: __RELEASE__
+    require_empty_target: false
     mirror: http://deb.debian.org/debian
     target: tag-root
     arch: __ARCH__
@@ -77,9 +78,9 @@ steps:
       - parted
       - ssh
       - wpasupplicant
-      - __SYSTEMD_TIMESYNCD__
+      - systemd-timesyncd
       - __LINUX_IMAGE__
-      - __RASPI_FIRMWARE__
+      - raspi-firmware
       - __WIRELESS_FIRMWARE__
       - __BLUETOOTH_FIRMWARE__
     tag: tag-root
@@ -112,7 +113,6 @@ steps:
       install -m 644 -o root -g root rootfs/etc/systemd/system/rpi-reconfigure-raspi-firmware.service "${ROOT?}/etc/systemd/system/"
       mkdir -p "${ROOT?}/etc/systemd/system/multi-user.target.requires/"
       ln -s /etc/systemd/system/rpi-reconfigure-raspi-firmware.service "${ROOT?}/etc/systemd/system/multi-user.target.requires/rpi-reconfigure-raspi-firmware.service"
-      __FIX_FIRMWARE_PKG_NAME__
 
       install -m 644 -o root -g root rootfs/etc/systemd/system/rpi-generate-ssh-host-keys.service "${ROOT?}/etc/systemd/system/"
       ln -s /etc/systemd/system/rpi-generate-ssh-host-keys.service "${ROOT?}/etc/systemd/system/multi-user.target.requires/rpi-generate-ssh-host-keys.service"
@@ -168,7 +168,7 @@ steps:
   - chroot: tag-root
     shell: |
       rm -f /etc/machine-id /var/lib/dbus/machine-id
-      __TOUCH_MACHINE_ID__
+      echo "uninitialized" > /etc/machine-id
 
   # Create /etc/raspi-image-id to know, from what commit the image was built
   - chroot: tag-root