From 48e4864fde427a04da2afa237584e7a032017570 Mon Sep 17 00:00:00 2001 From: Roland Clobus Date: Sat, 26 Oct 2024 10:42:57 +0200 Subject: [PATCH 01/12] Pipeline: turn off the arm64 pipeline for non-DDs The default arm64 (and riscv64) runners on Salsa are only available for DDs. Instead of enabling the runners for the main namespace, they are only turned off for non-DDs. See the documentation at https://salsa.debian.org/salsa-ci-team/pipeline#enable-building-on-arm-and-risc-v --- debian/gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/debian/gitlab-ci.yml b/debian/gitlab-ci.yml index 0b2d3d5d3..b0ea4f03b 100644 --- a/debian/gitlab-ci.yml +++ b/debian/gitlab-ci.yml @@ -47,6 +47,8 @@ ISO amd64: ISO arm64: extends: - .lb-build + rules: + - if: $CI_PROJECT_ROOT_NAMESPACE != "rclobus-guest" variables: BUILD_ARCH: 'arm64' tags: From bb7e9e63685adf7abd1a22557d43312851ec1c0d Mon Sep 17 00:00:00 2001 From: Roland Clobus Date: Sat, 26 Oct 2024 10:36:31 +0200 Subject: [PATCH 02/12] Workaround for #1084791 There is a file conflict in the non-free-firmware realtek packages, which prevent the sid live images from being built. This workaround will be removed a soon as the bug is resolved. --- functions/firmwarelists.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/functions/firmwarelists.sh b/functions/firmwarelists.sh index a79873ef9..9154ab919 100755 --- a/functions/firmwarelists.sh +++ b/functions/firmwarelists.sh @@ -90,6 +90,14 @@ Firmware_List_From_Contents () { then BLOCKLIST_FIRMWARE="${BLOCKLIST_FIRMWARE} amd64-microcode firmware-intel-sound firmware-sof-signed intel-microcode" fi + # Work around #1084791 (File conflict between firmware-realtek and firmware-realtek-rtl8723cs-bt) + # TODO: This work around must be removed as soon as the bug is fixed + if In_list non-free-firmware/kernel/firmware-realtek ${FIRMWARE_PACKAGES} \ + && In_list non-free-firmware/kernel/firmware-realtek-rtl8723cs-bt ${FIRMWARE_PACKAGES} \ + && In_list ${DISTRO_CHROOT} sid unstable + then + BLOCKLIST_FIRMWARE="${BLOCKLIST_FIRMWARE} firmware-realtek-rtl8723cs-bt" + fi # Deduplicate the list and prepare for easier manipulation by having each package on its own line local _FIRMWARE_PACKAGES_FILE=tmp_firmware_packages.txt From 399393565c4ff3ceba50ffbd88ecda102533b370 Mon Sep 17 00:00:00 2001 From: Roland Clobus Date: Mon, 28 Oct 2024 11:36:06 +0100 Subject: [PATCH 03/12] Workaround for #1084791 part 2 The previous commit did not work properly. Now output is generated that shows whether the workaround has been applied or needs to be removed again. --- functions/firmwarelists.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/functions/firmwarelists.sh b/functions/firmwarelists.sh index 9154ab919..d010f290a 100755 --- a/functions/firmwarelists.sh +++ b/functions/firmwarelists.sh @@ -92,11 +92,15 @@ Firmware_List_From_Contents () { fi # Work around #1084791 (File conflict between firmware-realtek and firmware-realtek-rtl8723cs-bt) # TODO: This work around must be removed as soon as the bug is fixed - if In_list non-free-firmware/kernel/firmware-realtek ${FIRMWARE_PACKAGES} \ - && In_list non-free-firmware/kernel/firmware-realtek-rtl8723cs-bt ${FIRMWARE_PACKAGES} \ - && In_list ${DISTRO_CHROOT} sid unstable + if In_list ${DISTRO_CHROOT} sid unstable then - BLOCKLIST_FIRMWARE="${BLOCKLIST_FIRMWARE} firmware-realtek-rtl8723cs-bt" + if In_list non-free-firmware/kernel/firmware-realtek,non-free-firmware/kernel/firmware-realtek-rtl8723cs-bt ${FIRMWARE_PACKAGES} + then + BLOCKLIST_FIRMWARE="${BLOCKLIST_FIRMWARE} firmware-realtek-rtl8723cs-bt" + Echo_message "Workaround for #1084791 has been applied by preferring the newer package firmware-realtek" + else + Echo_warning "Workaround for #1084791 must be removed again" + fi fi # Deduplicate the list and prepare for easier manipulation by having each package on its own line From 7349ab156fb538d42336b523b76400650418097c Mon Sep 17 00:00:00 2001 From: Aaron Rainbolt Date: Thu, 31 Oct 2024 00:05:45 -0500 Subject: [PATCH 04/12] Handle loopback booting with dracut-live properly dracut uses the `iso-scan/filename` kernel parameter for loopback booting with tools like Super Grub Disk, whereas live-boot uses the `findiso` parameter for this purpose. Conditionally set which one is used depending on the selected initramfs. --- scripts/build/binary_grub_cfg | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/scripts/build/binary_grub_cfg b/scripts/build/binary_grub_cfg index 087ca3595..71929839d 100755 --- a/scripts/build/binary_grub_cfg +++ b/scripts/build/binary_grub_cfg @@ -121,9 +121,22 @@ DEFAULT_INITRD="initrd.img-$(echo ${DEFAULT_KERNEL} | sed -e 's|vmlinuz-||')" KERNEL_LIVE="/${INITFS}/${DEFAULT_KERNEL}" INITRD_LIVE="/${INITFS}/${DEFAULT_INITRD}" -APPEND_LIVE="${LB_BOOTAPPEND_LIVE} findiso=\${iso_path}" FLAVOUR_LIVE="${DEFAULT_FLAVOUR}" +# live-boot and dracut use different kernel parameters for loopback +# booting +case "${LB_INITRAMFS}" in + live-boot) + APPEND_LIVE="${LB_BOOTAPPEND_LIVE} findiso=\${iso_path}" + ;; + dracut-live) + APPEND_LIVE="${LB_BOOTAPPEND_LIVE} iso-scan/filename=\${iso_path}" + ;; + none) + APPEND_LIVE="${LB_BOOTAPPEND_LIVE}" + ;; +esac + # Ensure fresh live entries LIVE_ENTRIES_TMP="${_TARGET}/live.cfg.tmp" rm -f "${LIVE_ENTRIES_TMP}" From 111a6c2c655b64cb637e9e477978fe5ca3a3cdaf Mon Sep 17 00:00:00 2001 From: Roland Clobus Date: Thu, 31 Oct 2024 17:59:15 +0100 Subject: [PATCH 05/12] Installer: use the http proxy --- functions/configuration.sh | 2 +- scripts/build/installer_debian-installer | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/functions/configuration.sh b/functions/configuration.sh index 4553ffb16..a122e771a 100755 --- a/functions/configuration.sh +++ b/functions/configuration.sh @@ -910,7 +910,7 @@ Validate_http_proxy () Validate_http_proxy_source "environment variable http_proxy" "${http_proxy}" Validate_http_proxy_source "command line option --apt-http-proxy" "${LB_APT_HTTP_PROXY}" - # This is the value to use for the the other scripts in live-build + # This is the value to use for the other scripts in live-build export http_proxy="${LAST_SEEN_PROXY_VALUE}" if [ ! -z "${http_proxy}" ]; then Echo_message "Using http proxy: ${http_proxy}" diff --git a/scripts/build/installer_debian-installer b/scripts/build/installer_debian-installer index 03edf72b8..96ee4a4c5 100755 --- a/scripts/build/installer_debian-installer +++ b/scripts/build/installer_debian-installer @@ -278,7 +278,7 @@ case "${LB_DERIVATIVE}" in # These variables do not need to be passed inside the chroot, they can be resolved earlier: # SOURCE_DATE_EPOCH, _QUIET, LB_PARENT_MIRROR_CHROOT, LB_PARENT_DISTRIBUTION_CHROOT - # TARGETS + # TARGETS, http_proxy cat << EOF > chroot/buildit.sh #!/bin/sh # Get the version of the git repo that matches SOURCE_DATE_EPOCH @@ -331,7 +331,7 @@ fi # USE_UDEBS_FROM -> use the same distribution as the chroot # ROOTCMD -> Workaround for #1058994. Fakeroot is not present in the chroot (and not required) export LINUX_KERNEL_ABI=\$(dpkg-query --showformat "\\\${db:Status-Abbrev} \\\${Package}\n" --show linux-image-* | awk '\$1=="ii" { c = split(\$2, a, "-"); if (c>3) { if (a[4] ~ /^[0-9]+/) { print a[3] "-" a[4] } else { print a[3] } } }' | sort | tail -1) -MIRROR="[check-valid-until=no] ${LB_PARENT_MIRROR_CHROOT}" TARGETS="${TARGETS}" USE_UDEBS_FROM=${LB_PARENT_DISTRIBUTION_CHROOT} ROOTCMD=" " bash ./daily-build build-only +http_proxy=${http_proxy} MIRROR="[check-valid-until=no] ${LB_PARENT_MIRROR_CHROOT}" TARGETS="${TARGETS}" USE_UDEBS_FROM=${LB_PARENT_DISTRIBUTION_CHROOT} ROOTCMD=" " bash ./daily-build build-only EOF Chroot chroot "sh buildit.sh" # Place the files in the cache. Download_file will use the cache instead of downloading From 0959dda89f7b48b9c469fefcd6eb7403edc5b4fc Mon Sep 17 00:00:00 2001 From: Roland Clobus Date: Sun, 27 Oct 2024 12:34:04 +0100 Subject: [PATCH 06/12] dracut: Support diverted configuration files of Calamares When a configuration file for Calamares is diverted by e.g. `config-package-dev`, it becomes a symlink. The in-place modifications must be applied on the symlinked file, not on a newly created copy. After removal of the package, no remainders are left. See https://salsa.debian.org/live-team/live-build/-/merge_requests/353#note_540394 --- share/hooks/normal/5050-dracut.hook.chroot | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/share/hooks/normal/5050-dracut.hook.chroot b/share/hooks/normal/5050-dracut.hook.chroot index 22065094e..60acd1676 100755 --- a/share/hooks/normal/5050-dracut.hook.chroot +++ b/share/hooks/normal/5050-dracut.hook.chroot @@ -29,17 +29,17 @@ apt-get autoremove --yes # Adjust the path for Calamares if [ -e /etc/calamares/modules/unpackfs.conf ] then - sed -i -e 's|/run/live/medium|/run/initramfs/live|' /etc/calamares/modules/unpackfs.conf + sed --follow-symlinks -i -e 's|/run/live/medium|/run/initramfs/live|' /etc/calamares/modules/unpackfs.conf fi # Use dracut instead of initramfs-tools if [ -e /etc/calamares/settings.conf ] then - sed -i -e '/initramfscfg/d;s/initramfs/dracut/' /etc/calamares/settings.conf + sed --follow-symlinks -i -e '/initramfscfg/d;s/initramfs/dracut/' /etc/calamares/settings.conf fi # Add dracut-live to the list of packages to uninstall if [ -e /etc/calamares/modules/packages.conf ] then - sed -i -e "s/'live-boot'/'dracut-live'/" /etc/calamares/modules/packages.conf + sed --follow-symlinks -i -e "s/'live-boot'/'dracut-live'/" /etc/calamares/modules/packages.conf fi # Calamares script for /etc/apt/sources.list during the installation SOURCES_MEDIA=/usr/share/calamares/helpers/calamares-sources-media From 30d39f812e41eb81928a0fe1f3a4f686eb30dfa9 Mon Sep 17 00:00:00 2001 From: Roland Clobus Date: Sat, 2 Nov 2024 16:01:06 +0100 Subject: [PATCH 07/12] lb config: --distribution defaults to testing --- functions/configuration.sh | 2 +- manpages/en/lb_config.1 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/functions/configuration.sh b/functions/configuration.sh index a122e771a..712398e37 100755 --- a/functions/configuration.sh +++ b/functions/configuration.sh @@ -41,7 +41,7 @@ Prepare_config () LB_MODE="${LB_MODE:-debian}" LB_DERIVATIVE="false" - LB_DISTRIBUTION="${LB_DISTRIBUTION:-bullseye}" + LB_DISTRIBUTION="${LB_DISTRIBUTION:-testing}" LB_DISTRIBUTION_CHROOT="${LB_DISTRIBUTION_CHROOT:-${LB_DISTRIBUTION}}" LB_DISTRIBUTION_BINARY="${LB_DISTRIBUTION_BINARY:-${LB_DISTRIBUTION_CHROOT}}" diff --git a/manpages/en/lb_config.1 b/manpages/en/lb_config.1 index be5520570..3a1aa1d49 100644 --- a/manpages/en/lb_config.1 +++ b/manpages/en/lb_config.1 @@ -331,7 +331,7 @@ tells debootstrap to use an alternate bootstrap script (last parameter to deboot .IP "\fB\-\-debug\fR" 4 turns on debugging informational messages. .IP "\fB\-d\fR|\fB\-\-distribution\fR \fICODENAME\fR" 4 -defines the distribution of the resulting live system. This currently defaults to 'buster'. The value 'sid' can be used for Debian unstable. +defines the distribution of the resulting live system. This defaults to 'testing'. The value 'sid' can be used for Debian unstable. .IP "\fB\-\-distribution\-binary\fR \fICODENAME\fR" 4 defines the distribution enabled in the resulting live system (defaults to the value set in \fB\-\-distribution\fR) .IP "\fB\-\-distribution\-chroot\fR \fICODENAME\fR" 4 From e4168673d188bde964b9b72a77932567f8229931 Mon Sep 17 00:00:00 2001 From: Roland Clobus Date: Sat, 2 Nov 2024 18:28:40 +0100 Subject: [PATCH 08/12] autopkgtest: Use 'stable' and 'testing' Use the stable names instead of the codenames to reduce maintenance of the autopkgtests. --- debian/tests/build-default-image | 8 +++++++- debian/tests/build-stable-image | 7 ++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/debian/tests/build-default-image b/debian/tests/build-default-image index a8824605c..10053e02e 100755 --- a/debian/tests/build-default-image +++ b/debian/tests/build-default-image @@ -4,6 +4,12 @@ set -eu set -o pipefail cd "${AUTOPKGTEST_TMP}" -lb config --verbose --updates false --security false +# Use the default values +lb config --verbose +# Verify some values +lb config --dump | grep 'LB_DISTRIBUTION="testing"' +lb config --dump | grep 'LB_UPDATES="true"' +lb config --dump | grep 'LB_SECURITY="true"' +lb config --dump | grep 'LB_PROPOSED_UPDATES="false"' lb build --verbose ls -l diff --git a/debian/tests/build-stable-image b/debian/tests/build-stable-image index 59b03a7da..68651582c 100755 --- a/debian/tests/build-stable-image +++ b/debian/tests/build-stable-image @@ -4,6 +4,11 @@ set -eu set -o pipefail cd "${AUTOPKGTEST_TMP}" -lb config --verbose --updates true --security true --distribution buster +# A minimal command line +lb config --verbose --distribution stable +# Verify some default values +lb config --dump | grep 'LB_UPDATES="true"' +lb config --dump | grep 'LB_SECURITY="true"' +lb config --dump | grep 'LB_PROPOSED_UPDATES="false"' lb build --verbose ls -l From 0bf71641ef9aaf63e69b8232636154cf9270852e Mon Sep 17 00:00:00 2001 From: Roland Clobus Date: Wed, 6 Nov 2024 11:31:14 +0100 Subject: [PATCH 09/12] man: Use $LIVE_BUILD if set The '--help' option will show the updated manpage if LIVE_BUILD is set --- functions/man.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/functions/man.sh b/functions/man.sh index f14858b97..701b4279f 100755 --- a/functions/man.sh +++ b/functions/man.sh @@ -12,7 +12,11 @@ Man () { if command -v man >/dev/null; then - man ${PROGRAM} + if [ -n "${LIVE_BUILD}" -a -e "${LIVE_BUILD}/manpages/en/lb_${PROGRAM#lb }.1" ]; then + man ${LIVE_BUILD}/manpages/en/lb_${PROGRAM#lb }.1 + else + man ${PROGRAM} + fi else Echo_warning "man is not installed, falling back to usage output." Usage From e6efd799fcdf0236986637206a6548eaf1f07f73 Mon Sep 17 00:00:00 2001 From: Roland Clobus Date: Wed, 6 Nov 2024 19:51:10 +0100 Subject: [PATCH 10/12] Workaround for #1084791 part 3 Package realtek-firmware 20240909-2 migrated to testing/trixie. The removal of firmware-realtek-rtl8723cs-bt in #1085075 has not taken place yet, so the workaround needs to be applied to trixie too. --- functions/firmwarelists.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/firmwarelists.sh b/functions/firmwarelists.sh index d010f290a..e169805ad 100755 --- a/functions/firmwarelists.sh +++ b/functions/firmwarelists.sh @@ -92,7 +92,7 @@ Firmware_List_From_Contents () { fi # Work around #1084791 (File conflict between firmware-realtek and firmware-realtek-rtl8723cs-bt) # TODO: This work around must be removed as soon as the bug is fixed - if In_list ${DISTRO_CHROOT} sid unstable + if In_list ${DISTRO_CHROOT} sid unstable trixie testing then if In_list non-free-firmware/kernel/firmware-realtek,non-free-firmware/kernel/firmware-realtek-rtl8723cs-bt ${FIRMWARE_PACKAGES} then From 6374459ad36eb3af2ac7541229e7369d0dd58855 Mon Sep 17 00:00:00 2001 From: Roland Clobus Date: Tue, 12 Nov 2024 09:34:02 +0100 Subject: [PATCH 11/12] Workaround for #1084791 removed. Source package `rtl8723bt-firmware` was removed from trixie 2024-11-12. The workaround is no longer required. --- functions/firmwarelists.sh | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/functions/firmwarelists.sh b/functions/firmwarelists.sh index e169805ad..a79873ef9 100755 --- a/functions/firmwarelists.sh +++ b/functions/firmwarelists.sh @@ -90,18 +90,6 @@ Firmware_List_From_Contents () { then BLOCKLIST_FIRMWARE="${BLOCKLIST_FIRMWARE} amd64-microcode firmware-intel-sound firmware-sof-signed intel-microcode" fi - # Work around #1084791 (File conflict between firmware-realtek and firmware-realtek-rtl8723cs-bt) - # TODO: This work around must be removed as soon as the bug is fixed - if In_list ${DISTRO_CHROOT} sid unstable trixie testing - then - if In_list non-free-firmware/kernel/firmware-realtek,non-free-firmware/kernel/firmware-realtek-rtl8723cs-bt ${FIRMWARE_PACKAGES} - then - BLOCKLIST_FIRMWARE="${BLOCKLIST_FIRMWARE} firmware-realtek-rtl8723cs-bt" - Echo_message "Workaround for #1084791 has been applied by preferring the newer package firmware-realtek" - else - Echo_warning "Workaround for #1084791 must be removed again" - fi - fi # Deduplicate the list and prepare for easier manipulation by having each package on its own line local _FIRMWARE_PACKAGES_FILE=tmp_firmware_packages.txt From cd7b87099582675cb0e02fa205fe95dca48cee09 Mon Sep 17 00:00:00 2001 From: Roland Clobus Date: Tue, 12 Nov 2024 12:26:06 +0100 Subject: [PATCH 12/12] Reproducible: ca-certificates-java This makes the postinst file from ca-certificates-java (/etc/ssl/certs/java/cacerts) reproducible. It contains embedded timestamps for 'now'. Debian-Junior includes ca-certificates-java and is now reproducible. --- ...roducible-ca-certificates-java.hook.chroot | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100755 examples/hooks/reproducible/2011-reproducible-ca-certificates-java.hook.chroot diff --git a/examples/hooks/reproducible/2011-reproducible-ca-certificates-java.hook.chroot b/examples/hooks/reproducible/2011-reproducible-ca-certificates-java.hook.chroot new file mode 100755 index 000000000..9f068b7e4 --- /dev/null +++ b/examples/hooks/reproducible/2011-reproducible-ca-certificates-java.hook.chroot @@ -0,0 +1,38 @@ +#!/bin/sh +set -e + +# /etc/ssl/certs/java/cacerts is a keystore +# When (re)generated, it embeds 'now' timestamps + +# Don't run if ca-certificates-java is not installed +if [ ! -e /etc/ssl/certs/java/cacerts ]; +then + exit 0 +fi + +# Use faketime to enforce a timestamp +# NB: hooks don't know about the apt/aptitude selection, so use 'apt-get' +export FAKETIME_ALREADY_INSTALLED=1 +if [ ! -e /usr/bin/faketime ]; +then + FAKETIME_ALREADY_INSTALLED=0 + apt-get install faketime --yes -o Acquire::Check-Valid-Until=false +fi + +# Remove the file +rm -f /etc/ssl/certs/java/cacerts + +# Generate it again +touch /var/lib/ca-certificates-java/fresh +# Java uses timestamps with millisecond resolution +# -f is required, otherwise the milliseconds are non-zero (due to relative timestamps) +faketime -f "$(date --utc -d@${SOURCE_DATE_EPOCH} +'%Y-%m-%d %H:%M:%SZ')" dpkg-reconfigure ca-certificates-java + +if [ ${FAKETIME_ALREADY_INSTALLED} -eq 0 ]; +then + apt-get remove --purge --yes faketime + apt-get autoremove --yes +fi + +echo "P: $(basename $0) Reproducible hook has been applied" +