diff --git a/Makefile b/Makefile index bae824f30..da2ce88cf 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ SHELL := sh -e LANGUAGES = $(shell cd manpages/po && ls) -SCRIPTS = frontend/* functions/* examples/auto/* examples/hooks/* scripts/*.sh scripts/*/* share/bin/* share/hooks/*/* +SCRIPTS = frontend/* functions/* examples/auto/* examples/hooks/*.chroot examples/hooks/reproducible/*.chroot scripts/*.sh scripts/*/* share/bin/* share/hooks/*/* all: build diff --git a/debian/control b/debian/control index 544eeeee3..a4b7ad5b3 100644 --- a/debian/control +++ b/debian/control @@ -8,6 +8,7 @@ Build-Depends: debhelper-compat (= 13), po4a, gettext, + devscripts, Standards-Version: 4.7.0 Rules-Requires-Root: no Homepage: https://wiki.debian.org/DebianLive diff --git a/functions/sourcelist.sh b/functions/sourcelist.sh index dedcc89db..4ea304522 100755 --- a/functions/sourcelist.sh +++ b/functions/sourcelist.sh @@ -28,8 +28,8 @@ Create_apt_sources_list () DISTRIBUTION=${LB_DISTRIBUTION_CHROOT} ;; binary) - PARENT_MIRROR=${LB_PARENT_MIRROR_BINARY} - MIRROR=${LB_MIRROR_BINARY} + PARENT_MIRROR="${LB_PARENT_MIRROR_BINARY}" + MIRROR="${LB_MIRROR_BINARY}" PARENT_MIRROR_SECURITY=${LB_PARENT_MIRROR_BINARY_SECURITY} MIRROR_SECURITY=${LB_MIRROR_BINARY_SECURITY} PARENT_DISTRIBUTION=${LB_PARENT_DISTRIBUTION_BINARY} diff --git a/test/rebuild.sh b/test/rebuild.sh index 5876ce8ea..6eadac94d 100755 --- a/test/rebuild.sh +++ b/test/rebuild.sh @@ -179,27 +179,27 @@ parse_commandline_arguments() { ;; "cinnamon") INSTALLER="live" - PACKAGES="live-task-cinnamon" + PACKAGES="live-task-cinnamon spice-vdagent" ;; "gnome") INSTALLER="live" - PACKAGES="live-task-gnome" + PACKAGES="live-task-gnome spice-vdagent" ;; "kde") INSTALLER="live" - PACKAGES="live-task-kde" + PACKAGES="live-task-kde spice-vdagent" ;; "lxde") INSTALLER="live" - PACKAGES="live-task-lxde" + PACKAGES="live-task-lxde spice-vdagent" ;; "lxqt") INSTALLER="live" - PACKAGES="live-task-lxqt" + PACKAGES="live-task-lxqt spice-vdagent" ;; "mate") INSTALLER="live" - PACKAGES="live-task-mate" + PACKAGES="live-task-mate spice-vdagent" ;; "standard") INSTALLER="live" @@ -207,11 +207,11 @@ parse_commandline_arguments() { ;; "xfce") INSTALLER="live" - PACKAGES="live-task-xfce" + PACKAGES="live-task-xfce spice-vdagent" ;; "debian-junior") INSTALLER="live" - PACKAGES="live-task-debian-junior" + PACKAGES="live-task-debian-junior spice-vdagent" ;; "") output_echo "Error: Missing --configuration" @@ -348,6 +348,23 @@ get_snapshot_from_archive() { rm latest } +get_snapshot_from_snapshot_debian_org() { + # Pick the snapshot closest to 'now' + wget ${WGET_OPTIONS} http://snapshot.debian.org/archive/debian/$(date --utc +%Y%m%dT%H%M%SZ)/dists/${DEBIAN_VERSION}/InRelease --output-document latest + # + # Extract the timestamp from the InRelease file + # + # Input: + # ... + # Date: Sat, 23 Jul 2022 14:33:45 UTC + # ... + # Output: + # 20220723T143345Z + # + SNAPSHOT_TIMESTAMP=$(cat latest | awk '/^Date:/ { print substr($0, 7) }' | xargs -I this_date date --utc --date "this_date" +%Y%m%dT%H%M%SZ) + rm latest +} + # # main: follow https://wiki.debian.org/ReproducibleInstalls/LiveImages # @@ -393,29 +410,21 @@ case ${BUILD_LATEST} in # Use the timestamp of the current Debian archive get_snapshot_from_archive MIRROR=http://deb.debian.org/debian/ + MIRROR_BINARY=${MIRROR} + MODIFY_APT_OPTIONS=0 ;; "snapshot") # Use the timestamp of the latest mirror snapshot - wget ${WGET_OPTIONS} http://snapshot.notset.fr/mr/timestamp/debian/latest --output-document latest - # - # Extract the timestamp from the JSON file - # - # Input: - # { - # "_api": "0.3", - # "_comment": "notset", - # "result": "20210828T083909Z" - # } - # Output: - # 20210828T083909Z - # - SNAPSHOT_TIMESTAMP=$(cat latest | awk '/"result":/ { split($0, a, "\""); print a[4] }') - rm latest - MIRROR=http://snapshot.notset.fr/archive/debian/${SNAPSHOT_TIMESTAMP} + get_snapshot_from_snapshot_debian_org + MIRROR=http://snapshot.debian.org/archive/debian/${SNAPSHOT_TIMESTAMP} + MIRROR_BINARY="[check-valid-until=no] ${MIRROR}" + MODIFY_APT_OPTIONS=1 ;; "no") # The value of SNAPSHOT_TIMESTAMP was provided on the command line - MIRROR=http://snapshot.notset.fr/archive/debian/${SNAPSHOT_TIMESTAMP} + MIRROR=http://snapshot.debian.org/archive/debian/${SNAPSHOT_TIMESTAMP} + MIRROR_BINARY="[check-valid-until=no] ${MIRROR}" + MODIFY_APT_OPTIONS=1 ;; *) echo "E: A new option to BUILD_LATEST has been added" @@ -457,7 +466,7 @@ fi output_echo "Running lb config." lb config \ --mirror-bootstrap ${MIRROR} \ - --mirror-binary ${MIRROR} \ + --mirror-binary "${MIRROR_BINARY}" \ --security false \ --updates false \ --distribution ${DEBIAN_VERSION} \ @@ -471,9 +480,11 @@ lb config \ ${GENERATE_SOURCE} \ 2>&1 | tee $LB_OUTPUT -# Insider knowledge of live-build: -# Add '-o Acquire::Check-Valid-Until=false', to allow for rebuilds of older timestamps -sed -i -e '/^APT_OPTIONS=/s/--yes/--yes -o Acquire::Check-Valid-Until=false/' config/common +if [ ${MODIFY_APT_OPTIONS} -ne 0 ]; then + # Insider knowledge of live-build: + # Add '-o Acquire::Check-Valid-Until=false', to allow for rebuilds of older timestamps + sed -i -e '/^APT_OPTIONS=/s/--yes/--yes -o Acquire::Check-Valid-Until=false/' config/common +fi if [ ! -z "${PACKAGES}" ]; then echo "${PACKAGES}" >config/package-lists/desktop.list.chroot @@ -634,17 +645,22 @@ fi # For oldstable and stable use the same boot splash screen as the Debian installer case "$DEBIAN_VERSION" in -"bullseye") +"bullseye"|"oldstable") mkdir -p config/bootloaders wget --quiet https://salsa.debian.org/installer-team/debian-installer/-/raw/master/build/boot/artwork/11-homeworld/homeworld.svg -O config/bootloaders/splash.svg mkdir -p config/bootloaders/grub-pc # Use the old resolution of 640x480 for grub ln -s ../../isolinux/splash.png config/bootloaders/grub-pc/splash.png ;; -"bookworm") +"bookworm"|"stable") mkdir -p config/bootloaders wget --quiet https://salsa.debian.org/installer-team/debian-installer/-/raw/master/build/boot/artwork/12-emerald/emerald.svg -O config/bootloaders/splash.svg ;; +"trixie"|"testing") + # Trixie artwork: https://wiki.debian.org/DebianArt/Themes/Ceratopsian + mkdir -p config/bootloaders + wget --quiet https://raw.githubusercontent.com/pccouper/trixie/refs/heads/main/grub/grub.svg -O config/bootloaders/splash.svg + ;; *) # Use the default 'under construction' image ;; @@ -659,8 +675,8 @@ BUILD_RESULT=$? set -e if [ ${BUILD_RESULT} -ne 0 ]; then # Find the snapshot that matches 1 second before the current snapshot - wget ${WGET_OPTIONS} http://snapshot.notset.fr/mr/timestamp/debian/$(date --utc -d @$((${SOURCE_DATE_EPOCH} - 1)) +%Y%m%dT%H%M%SZ) --output-document but_latest - PROPOSED_SNAPSHOT_TIMESTAMP=$(cat but_latest | awk '/"result":/ { split($0, a, "\""); print a[4] }') + wget ${WGET_OPTIONS} http://snapshot.debian.org/archive/debian/$(date --utc -d @$((${SOURCE_DATE_EPOCH} - 1)) +%Y%m%dT%H%M%SZ)/dists/${DEBIAN_VERSION}/InRelease --output-document but_latest + PROPOSED_SNAPSHOT_TIMESTAMP=$(cat but_latest | awk '/^Date:/ { print substr($0, 7) }' | xargs -I this_date date --utc --date "this_date" +%Y%m%dT%H%M%SZ) rm but_latest output_echo "Warning: lb build failed with ${BUILD_RESULT}. The latest snapshot might not be complete (yet). Try re-running the script with SNAPSHOT_TIMESTAMP=${PROPOSED_SNAPSHOT_TIMESTAMP}." diff --git a/test/test_external_deb_sources.sh b/test/test_external_deb_sources.sh index 44bf296b3..8d040c467 100755 --- a/test/test_external_deb_sources.sh +++ b/test/test_external_deb_sources.sh @@ -14,6 +14,11 @@ if ! command -v shunit2 > /dev/null; then exit 1 fi +if ! command -v faketime > /dev/null; then + echo "Install faketime" + exit 1 +fi + function create_packages () { # Create package generator files cat << EOF > package @@ -33,7 +38,7 @@ Description: Test package for testing the inclusion in live images Tests dependency chain Package live-testpackage-$1-dependency should be automatically installed and removed too EOF - equivs-build package + faketime -f "$(date --utc -d@${SOURCE_DATE_EPOCH} +'%Y-%m-%d %H:%M:%SZ')" equivs-build package cat << EOF > package Source: live-testpackage-$1-dependency @@ -51,7 +56,7 @@ Description: Test package for testing the inclusion in live images Tests dependency chain This package should be automatically installed and removed too EOF - equivs-build package + faketime -f "$(date --utc -d@${SOURCE_DATE_EPOCH} +'%Y-%m-%d %H:%M:%SZ')" equivs-build package rm package } @@ -96,8 +101,8 @@ function setUp() { # Create a test configuration lb clean --purge rm -fr config - # Slight speedup: --zsync, --firmware-chroot - lb config --distribution unstable --zsync false --firmware-chroot false + # Slight speedup: --zsync, --firmware-chroot, --cache + lb config --distribution unstable --zsync false --firmware-chroot false --cache false } function build_image() { @@ -105,14 +110,18 @@ function build_image() { export MKSQUASHFS_OPTIONS=-no-compression # Perform the build lb build + if [ -e live-image-amd64.hybrid.iso ] + then + sha256sum --tag live-image-amd64.hybrid.iso + fi } function test_snapshot_with_mirror_bootstrap() { # Rebuild the configuration, as many mirror settings depend on eachother lb clean --purge rm -fr config - # Slight speedup: --zsync, --firmware-chroot - lb config --distribution unstable --zsync false --firmware-chroot false --mirror-bootstrap http://snapshot.debian.org/archive/debian/20240701T000000Z/ --mirror-binary http://deb.debian.org/debian/ + # Slight speedup: --zsync, --firmware-chroot, --cache + lb config --distribution unstable --zsync false --firmware-chroot false --cache false --mirror-bootstrap http://snapshot.debian.org/archive/debian/20240701T000000Z/ --mirror-binary http://deb.debian.org/debian/ # Insider knowledge of live-build: # Add '-o Acquire::Check-Valid-Until=false', to allow for rebuilds of older timestamps sed -i -e '/^APT_OPTIONS=/s/--yes/--yes -o Acquire::Check-Valid-Until=false/' config/common @@ -125,6 +134,107 @@ function test_snapshot_with_mirror_bootstrap() { unmountSquashfs } +function test_preexisting_package_inclusion_chroot() { + # Why this package? + # - It has only a few dependencies + # - It is not present in the small image + echo "hwdata" > config/package-lists/config-package-lists-chroot.list.chroot + build_image + assertTrue "Main package is installed (install)" "grep -q '^hwdata' chroot.packages.install" + assertTrue "Dependency package is installed (install)" "grep -q '^pci.ids' chroot.packages.install" + assertTrue "Main package is installed (live)" "grep -q '^hwdata' chroot.packages.live" + assertTrue "Dependency package is installed (live)" "grep -q '^pci.ids' chroot.packages.live" + mountSquashfs + assertFalse "Main package stays after installation" "grep -q '^hwdata' iso/live/filesystem.packages-remove" + assertFalse "Dependency package stays after installation" "grep -q '^pci\.ids' iso/live/filesystem.packages-remove" + assertFalse "No package pool should be generated" "[ -e iso/pool ]" + assertFalse "Package pool is not listed in /etc/apt/sources.list" "grep -q 'file:/run/live/medium' squashfs/etc/apt/sources.list" + unmountSquashfs +} + +function test_preexisting_package_inclusion_chroot_live() { + # Why this package? + # - It has only a few dependencies + # - It is not present in the small image + echo "hwdata" > config/package-lists/config-package-lists-chroot-live.list.chroot_live + build_image + assertFalse "Main package is not installed (install)" "grep -q '^hwdata' chroot.packages.install" + assertFalse "Dependency package is not installed (install)" "grep -q '^pci.ids' chroot.packages.install" + assertTrue "Main package is installed (live)" "grep -q '^hwdata' chroot.packages.live" + assertTrue "Dependency package is installed (live)" "grep -q '^pci.ids' chroot.packages.live" + mountSquashfs + assertTrue "Main package will be removed after installation" "grep -q '^hwdata' iso/live/filesystem.packages-remove" + assertTrue "Dependency package will be removed after installation" "grep -q '^pci\.ids' iso/live/filesystem.packages-remove" + assertFalse "No package pool should be generated" "[ -e iso/pool ]" + assertFalse "Package pool is not listed in /etc/apt/sources.list" "grep -q 'file:/run/live/medium' squashfs/etc/apt/sources.list" + unmountSquashfs +} + +# Effectively is a duplicate of test_preexisting_package_inclusion_chroot +function test_preexisting_package_inclusion_chroot_install() { + # Why this package? + # - It has only a few dependencies + # - It is not present in the small image + echo "hwdata" > config/package-lists/config-package-lists-chroot-live.list.chroot_install + build_image + assertTrue "Main package is installed (install)" "grep -q '^hwdata' chroot.packages.install" + assertTrue "Dependency package is installed (install)" "grep -q '^pci.ids' chroot.packages.install" + assertTrue "Main package is installed (live)" "grep -q '^hwdata' chroot.packages.live" + assertTrue "Dependency package is installed (live)" "grep -q '^pci.ids' chroot.packages.live" + mountSquashfs + assertFalse "Main package stays after installation" "grep -q '^hwdata' iso/live/filesystem.packages-remove" + assertFalse "Dependency package stays after installation" "grep -q '^pci\.ids' iso/live/filesystem.packages-remove" + assertFalse "No package pool should be generated" "[ -e iso/pool ]" + assertFalse "Package pool is not listed in /etc/apt/sources.list" "grep -q 'file:/run/live/medium' squashfs/etc/apt/sources.list" + unmountSquashfs +} + +function test_preexisting_package_inclusion_unspecified_chroot_or_binary() { + # Why this package? + # - It has only a few dependencies + # - It is not present in the small image + echo "hwdata" > config/package-lists/config-package-lists-chroot.list + build_image + assertTrue "Main package is installed (install)" "grep -q '^hwdata' chroot.packages.install" + assertTrue "Dependency package is installed (install)" "grep -q '^pci.ids' chroot.packages.install" + assertTrue "Main package is installed (live)" "grep -q '^hwdata' chroot.packages.live" + assertTrue "Dependency package is installed (live)" "grep -q '^pci.ids' chroot.packages.live" + mountSquashfs + assertFalse "Main package stays after installation" "grep -q '^hwdata' iso/live/filesystem.packages-remove" + assertFalse "Dependency package stays after installation" "grep -q '^pci\.ids' iso/live/filesystem.packages-remove" + assertTrue "Main package should be in the pool" "[ -e iso/pool/main/h/hwdata/hwdata_*_all.deb ]" + assertTrue "Dependency package should be in the pool" "[ -e iso/pool/main/p/pci.ids/pci.ids_*_all.deb ]" + assertTrue "Package pool is listed in /etc/apt/sources.list" "grep -q 'file:/run/live/medium' squashfs/etc/apt/sources.list" + assertTrue "Sources list meta info should be present: Release" "[ -e squashfs/var/lib/apt/lists/_run_live_medium_dists_unstable_Release ]" + assertTrue "Sources list meta info should be present: Packages" "[ -e squashfs/var/lib/apt/lists/_run_live_medium_dists_unstable_main_binary-amd64_Packages ]" + unmountSquashfs +} + +function test_preexisting_package_inclusion_binary() { + # Why this package? + # - It has only a few dependencies + # - It is not present in the small image + echo "hwdata" > config/package-lists/config-package-lists-chroot.list.binary + build_image + assertFalse "Main package is not installed (install)" "grep -q '^hwdata' chroot.packages.install" + assertFalse "Dependency package is not installed (install)" "grep -q '^pci.ids' chroot.packages.install" + assertFalse "Main package is not installed (live)" "grep -q '^hwdata' chroot.packages.live" + assertFalse "Dependency package is not installed (live)" "grep -q '^pci.ids' chroot.packages.live" + mountSquashfs + assertFalse "Main package stays after installation" "grep -q '^hwdata' iso/live/filesystem.packages-remove" + assertFalse "Dependency package stays after installation" "grep -q '^pci\.ids' iso/live/filesystem.packages-remove" + assertTrue "Main package should be in the pool" "[ -e iso/pool/main/h/hwdata/hwdata_*_all.deb ]" + assertTrue "Dependency package should be in the pool" "[ -e iso/pool/main/p/pci.ids/pci.ids_*_all.deb ]" + assertTrue "Package pool is listed in /etc/apt/sources.list" "grep -q 'file:/run/live/medium' squashfs/etc/apt/sources.list" + assertTrue "Sources list meta info should be present: Release" "[ -e squashfs/var/lib/apt/lists/_run_live_medium_dists_unstable_Release ]" + assertTrue "Sources list meta info should be present: Packages" "[ -e squashfs/var/lib/apt/lists/_run_live_medium_dists_unstable_main_binary-amd64_Packages ]" + unmountSquashfs +} + +# Untested: +# 8.2.5 Generated package lists +# 8.2.6 Using conditionals inside package lists + function test_direct_inclusion_of_deb_unspecified_chroot_or_binary() { create_packages config-packages cp live-testpackage-config-packages-main_1.0_all.deb config/packages @@ -155,8 +265,8 @@ function test_direct_inclusion_of_deb_binary() { assertTrue "Main package should be in the pool" "[ -e iso/pool/main/l/live-testpackage-config-packages-binary-main/live-testpackage-config-packages-binary-main_1.0_all.deb ]" assertTrue "Dependency package should be in the pool" "[ -e iso/pool/main/l/live-testpackage-config-packages-binary-dependency/live-testpackage-config-packages-binary-dependency_1.0_all.deb ]" assertTrue "Package pool is listed in /etc/apt/sources.list" "grep -q 'file:/run/live/medium' squashfs/etc/apt/sources.list" - assertTrue "Sources list meta info should be present" "[ -e squashfs/var/lib/apt/lists/_run_live_medium_dists_unstable_Release ]" - assertTrue "Sources list meta info should be present" "[ -e squashfs/var/lib/apt/lists/_run_live_medium_dists_unstable_main_binary-amd64_Packages ]" + assertTrue "Sources list meta info should be present: Release" "[ -e squashfs/var/lib/apt/lists/_run_live_medium_dists_unstable_Release ]" + assertTrue "Sources list meta info should be present: Packages" "[ -e squashfs/var/lib/apt/lists/_run_live_medium_dists_unstable_main_binary-amd64_Packages ]" unmountSquashfs } @@ -314,8 +424,8 @@ EOF assertTrue "Main package should be in the pool" "[ -e iso/pool/main/l/live-testpackage-config-archives-list-binary-main/live-testpackage-config-archives-list-binary-main_1.0_all.deb ]" assertTrue "Dependency package should be in the pool" "[ -e iso/pool/main/l/live-testpackage-config-archives-list-binary-dependency/live-testpackage-config-archives-list-binary-dependency_1.0_all.deb ]" assertTrue "Package pool is listed in /etc/apt/sources.list" "grep -q 'file:/run/live/medium' squashfs/etc/apt/sources.list" - assertTrue "Sources list meta info should be present" "[ -e squashfs/var/lib/apt/lists/_run_live_medium_dists_unstable_Release ]" - assertTrue "Sources list meta info should be present" "[ -e squashfs/var/lib/apt/lists/_run_live_medium_dists_unstable_main_binary-amd64_Packages ]" + assertTrue "Sources list meta info should be present: Release" "[ -e squashfs/var/lib/apt/lists/_run_live_medium_dists_unstable_Release ]" + assertTrue "Sources list meta info should be present: Packages" "[ -e squashfs/var/lib/apt/lists/_run_live_medium_dists_unstable_main_binary-amd64_Packages ]" unmountSquashfs } @@ -351,8 +461,8 @@ function test_derivatives() { # Rebuild the configuration, as many mirror settings depend on eachother #lb clean --purge #rm -fr config - # Slight speedup: --zsync, --firmware-chroot - #lb config --distribution unstable --zsync false --firmware-chroot false + # Slight speedup: --zsync, --firmware-chroot, --cache + #lb config --distribution unstable --zsync false --firmware-chroot false --cache false # Let's not test --parent-distribution-chroot at the moment: # --apt-secure false --parent-mirror-chroot file://localhost$(pwd)/testrepository --parent-distribution-chroot nondebian --parent-archive-areas mymain --mirror-chroot http://deb.debian.org/debian --distribution-chroot debian --archive-areas main --parent-mirror-bootstrap file://localhost$(pwd)/testrepository # --apt-secure false --mirror-chroot file://localhost$(pwd)/testrepository-mirror-chroot --distribution-chroot nondebian --archive-areas mymain --parent-mirror-chroot http://deb.debian.org/debian --parent-distribution-chroot unstable --parent-archive-areas main @@ -363,4 +473,7 @@ function test_derivatives() { #unmountSquashfs } -. shunit2 +SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(date --utc '+%s')}" +ISO8601_TIMESTAMP=$(date --utc -d@${SOURCE_DATE_EPOCH} +%Y-%m-%dT%H:%M:%SZ) +. shunit2 2> logfile_${ISO8601_TIMESTAMP}.stderr | tee logfile_${ISO8601_TIMESTAMP}.stdout +egrep "ASSERT|FAILED|OK|shunit2|test_|SHA256" logfile_${ISO8601_TIMESTAMP}.stdout | tee logfile_${ISO8601_TIMESTAMP}.summary