Merge branch 'rclobus/various' into 'master'

Various patches

See merge request live-team/live-build!396
This commit is contained in:
Roland Clobus 2025-01-08 11:12:38 +00:00
commit b204c5bb5e
5 changed files with 179 additions and 49 deletions

View File

@ -4,7 +4,7 @@ SHELL := sh -e
LANGUAGES = $(shell cd manpages/po && ls)
SCRIPTS = frontend/* functions/* examples/auto/* examples/hooks/* scripts/*.sh scripts/*/* share/bin/* share/hooks/*/*
SCRIPTS = frontend/* functions/* examples/auto/* examples/hooks/*.chroot examples/hooks/reproducible/*.chroot scripts/*.sh scripts/*/* share/bin/* share/hooks/*/*
all: build

1
debian/control vendored
View File

@ -8,6 +8,7 @@ Build-Depends:
debhelper-compat (= 13),
po4a,
gettext,
devscripts,
Standards-Version: 4.7.0
Rules-Requires-Root: no
Homepage: https://wiki.debian.org/DebianLive

View File

@ -28,8 +28,8 @@ Create_apt_sources_list ()
DISTRIBUTION=${LB_DISTRIBUTION_CHROOT}
;;
binary)
PARENT_MIRROR=${LB_PARENT_MIRROR_BINARY}
MIRROR=${LB_MIRROR_BINARY}
PARENT_MIRROR="${LB_PARENT_MIRROR_BINARY}"
MIRROR="${LB_MIRROR_BINARY}"
PARENT_MIRROR_SECURITY=${LB_PARENT_MIRROR_BINARY_SECURITY}
MIRROR_SECURITY=${LB_MIRROR_BINARY_SECURITY}
PARENT_DISTRIBUTION=${LB_PARENT_DISTRIBUTION_BINARY}

View File

@ -179,27 +179,27 @@ parse_commandline_arguments() {
;;
"cinnamon")
INSTALLER="live"
PACKAGES="live-task-cinnamon"
PACKAGES="live-task-cinnamon spice-vdagent"
;;
"gnome")
INSTALLER="live"
PACKAGES="live-task-gnome"
PACKAGES="live-task-gnome spice-vdagent"
;;
"kde")
INSTALLER="live"
PACKAGES="live-task-kde"
PACKAGES="live-task-kde spice-vdagent"
;;
"lxde")
INSTALLER="live"
PACKAGES="live-task-lxde"
PACKAGES="live-task-lxde spice-vdagent"
;;
"lxqt")
INSTALLER="live"
PACKAGES="live-task-lxqt"
PACKAGES="live-task-lxqt spice-vdagent"
;;
"mate")
INSTALLER="live"
PACKAGES="live-task-mate"
PACKAGES="live-task-mate spice-vdagent"
;;
"standard")
INSTALLER="live"
@ -207,11 +207,11 @@ parse_commandline_arguments() {
;;
"xfce")
INSTALLER="live"
PACKAGES="live-task-xfce"
PACKAGES="live-task-xfce spice-vdagent"
;;
"debian-junior")
INSTALLER="live"
PACKAGES="live-task-debian-junior"
PACKAGES="live-task-debian-junior spice-vdagent"
;;
"")
output_echo "Error: Missing --configuration"
@ -348,6 +348,23 @@ get_snapshot_from_archive() {
rm latest
}
get_snapshot_from_snapshot_debian_org() {
# Pick the snapshot closest to 'now'
wget ${WGET_OPTIONS} http://snapshot.debian.org/archive/debian/$(date --utc +%Y%m%dT%H%M%SZ)/dists/${DEBIAN_VERSION}/InRelease --output-document latest
#
# Extract the timestamp from the InRelease file
#
# Input:
# ...
# Date: Sat, 23 Jul 2022 14:33:45 UTC
# ...
# Output:
# 20220723T143345Z
#
SNAPSHOT_TIMESTAMP=$(cat latest | awk '/^Date:/ { print substr($0, 7) }' | xargs -I this_date date --utc --date "this_date" +%Y%m%dT%H%M%SZ)
rm latest
}
#
# main: follow https://wiki.debian.org/ReproducibleInstalls/LiveImages
#
@ -393,29 +410,21 @@ case ${BUILD_LATEST} in
# Use the timestamp of the current Debian archive
get_snapshot_from_archive
MIRROR=http://deb.debian.org/debian/
MIRROR_BINARY=${MIRROR}
MODIFY_APT_OPTIONS=0
;;
"snapshot")
# Use the timestamp of the latest mirror snapshot
wget ${WGET_OPTIONS} http://snapshot.notset.fr/mr/timestamp/debian/latest --output-document latest
#
# Extract the timestamp from the JSON file
#
# Input:
# {
# "_api": "0.3",
# "_comment": "notset",
# "result": "20210828T083909Z"
# }
# Output:
# 20210828T083909Z
#
SNAPSHOT_TIMESTAMP=$(cat latest | awk '/"result":/ { split($0, a, "\""); print a[4] }')
rm latest
MIRROR=http://snapshot.notset.fr/archive/debian/${SNAPSHOT_TIMESTAMP}
get_snapshot_from_snapshot_debian_org
MIRROR=http://snapshot.debian.org/archive/debian/${SNAPSHOT_TIMESTAMP}
MIRROR_BINARY="[check-valid-until=no] ${MIRROR}"
MODIFY_APT_OPTIONS=1
;;
"no")
# The value of SNAPSHOT_TIMESTAMP was provided on the command line
MIRROR=http://snapshot.notset.fr/archive/debian/${SNAPSHOT_TIMESTAMP}
MIRROR=http://snapshot.debian.org/archive/debian/${SNAPSHOT_TIMESTAMP}
MIRROR_BINARY="[check-valid-until=no] ${MIRROR}"
MODIFY_APT_OPTIONS=1
;;
*)
echo "E: A new option to BUILD_LATEST has been added"
@ -457,7 +466,7 @@ fi
output_echo "Running lb config."
lb config \
--mirror-bootstrap ${MIRROR} \
--mirror-binary ${MIRROR} \
--mirror-binary "${MIRROR_BINARY}" \
--security false \
--updates false \
--distribution ${DEBIAN_VERSION} \
@ -471,9 +480,11 @@ lb config \
${GENERATE_SOURCE} \
2>&1 | tee $LB_OUTPUT
# Insider knowledge of live-build:
# Add '-o Acquire::Check-Valid-Until=false', to allow for rebuilds of older timestamps
sed -i -e '/^APT_OPTIONS=/s/--yes/--yes -o Acquire::Check-Valid-Until=false/' config/common
if [ ${MODIFY_APT_OPTIONS} -ne 0 ]; then
# Insider knowledge of live-build:
# Add '-o Acquire::Check-Valid-Until=false', to allow for rebuilds of older timestamps
sed -i -e '/^APT_OPTIONS=/s/--yes/--yes -o Acquire::Check-Valid-Until=false/' config/common
fi
if [ ! -z "${PACKAGES}" ]; then
echo "${PACKAGES}" >config/package-lists/desktop.list.chroot
@ -634,17 +645,22 @@ fi
# For oldstable and stable use the same boot splash screen as the Debian installer
case "$DEBIAN_VERSION" in
"bullseye")
"bullseye"|"oldstable")
mkdir -p config/bootloaders
wget --quiet https://salsa.debian.org/installer-team/debian-installer/-/raw/master/build/boot/artwork/11-homeworld/homeworld.svg -O config/bootloaders/splash.svg
mkdir -p config/bootloaders/grub-pc
# Use the old resolution of 640x480 for grub
ln -s ../../isolinux/splash.png config/bootloaders/grub-pc/splash.png
;;
"bookworm")
"bookworm"|"stable")
mkdir -p config/bootloaders
wget --quiet https://salsa.debian.org/installer-team/debian-installer/-/raw/master/build/boot/artwork/12-emerald/emerald.svg -O config/bootloaders/splash.svg
;;
"trixie"|"testing")
# Trixie artwork: https://wiki.debian.org/DebianArt/Themes/Ceratopsian
mkdir -p config/bootloaders
wget --quiet https://raw.githubusercontent.com/pccouper/trixie/refs/heads/main/grub/grub.svg -O config/bootloaders/splash.svg
;;
*)
# Use the default 'under construction' image
;;
@ -659,8 +675,8 @@ BUILD_RESULT=$?
set -e
if [ ${BUILD_RESULT} -ne 0 ]; then
# Find the snapshot that matches 1 second before the current snapshot
wget ${WGET_OPTIONS} http://snapshot.notset.fr/mr/timestamp/debian/$(date --utc -d @$((${SOURCE_DATE_EPOCH} - 1)) +%Y%m%dT%H%M%SZ) --output-document but_latest
PROPOSED_SNAPSHOT_TIMESTAMP=$(cat but_latest | awk '/"result":/ { split($0, a, "\""); print a[4] }')
wget ${WGET_OPTIONS} http://snapshot.debian.org/archive/debian/$(date --utc -d @$((${SOURCE_DATE_EPOCH} - 1)) +%Y%m%dT%H%M%SZ)/dists/${DEBIAN_VERSION}/InRelease --output-document but_latest
PROPOSED_SNAPSHOT_TIMESTAMP=$(cat but_latest | awk '/^Date:/ { print substr($0, 7) }' | xargs -I this_date date --utc --date "this_date" +%Y%m%dT%H%M%SZ)
rm but_latest
output_echo "Warning: lb build failed with ${BUILD_RESULT}. The latest snapshot might not be complete (yet). Try re-running the script with SNAPSHOT_TIMESTAMP=${PROPOSED_SNAPSHOT_TIMESTAMP}."

View File

@ -14,6 +14,11 @@ if ! command -v shunit2 > /dev/null; then
exit 1
fi
if ! command -v faketime > /dev/null; then
echo "Install faketime"
exit 1
fi
function create_packages () {
# Create package generator files
cat << EOF > package
@ -33,7 +38,7 @@ Description: Test package for testing the inclusion in live images
Tests dependency chain
Package live-testpackage-$1-dependency should be automatically installed and removed too
EOF
equivs-build package
faketime -f "$(date --utc -d@${SOURCE_DATE_EPOCH} +'%Y-%m-%d %H:%M:%SZ')" equivs-build package
cat << EOF > package
Source: live-testpackage-$1-dependency
@ -51,7 +56,7 @@ Description: Test package for testing the inclusion in live images
Tests dependency chain
This package should be automatically installed and removed too
EOF
equivs-build package
faketime -f "$(date --utc -d@${SOURCE_DATE_EPOCH} +'%Y-%m-%d %H:%M:%SZ')" equivs-build package
rm package
}
@ -96,8 +101,8 @@ function setUp() {
# Create a test configuration
lb clean --purge
rm -fr config
# Slight speedup: --zsync, --firmware-chroot
lb config --distribution unstable --zsync false --firmware-chroot false
# Slight speedup: --zsync, --firmware-chroot, --cache
lb config --distribution unstable --zsync false --firmware-chroot false --cache false
}
function build_image() {
@ -105,14 +110,18 @@ function build_image() {
export MKSQUASHFS_OPTIONS=-no-compression
# Perform the build
lb build
if [ -e live-image-amd64.hybrid.iso ]
then
sha256sum --tag live-image-amd64.hybrid.iso
fi
}
function test_snapshot_with_mirror_bootstrap() {
# Rebuild the configuration, as many mirror settings depend on eachother
lb clean --purge
rm -fr config
# Slight speedup: --zsync, --firmware-chroot
lb config --distribution unstable --zsync false --firmware-chroot false --mirror-bootstrap http://snapshot.debian.org/archive/debian/20240701T000000Z/ --mirror-binary http://deb.debian.org/debian/
# Slight speedup: --zsync, --firmware-chroot, --cache
lb config --distribution unstable --zsync false --firmware-chroot false --cache false --mirror-bootstrap http://snapshot.debian.org/archive/debian/20240701T000000Z/ --mirror-binary http://deb.debian.org/debian/
# Insider knowledge of live-build:
# Add '-o Acquire::Check-Valid-Until=false', to allow for rebuilds of older timestamps
sed -i -e '/^APT_OPTIONS=/s/--yes/--yes -o Acquire::Check-Valid-Until=false/' config/common
@ -125,6 +134,107 @@ function test_snapshot_with_mirror_bootstrap() {
unmountSquashfs
}
function test_preexisting_package_inclusion_chroot() {
# Why this package?
# - It has only a few dependencies
# - It is not present in the small image
echo "hwdata" > config/package-lists/config-package-lists-chroot.list.chroot
build_image
assertTrue "Main package is installed (install)" "grep -q '^hwdata' chroot.packages.install"
assertTrue "Dependency package is installed (install)" "grep -q '^pci.ids' chroot.packages.install"
assertTrue "Main package is installed (live)" "grep -q '^hwdata' chroot.packages.live"
assertTrue "Dependency package is installed (live)" "grep -q '^pci.ids' chroot.packages.live"
mountSquashfs
assertFalse "Main package stays after installation" "grep -q '^hwdata' iso/live/filesystem.packages-remove"
assertFalse "Dependency package stays after installation" "grep -q '^pci\.ids' iso/live/filesystem.packages-remove"
assertFalse "No package pool should be generated" "[ -e iso/pool ]"
assertFalse "Package pool is not listed in /etc/apt/sources.list" "grep -q 'file:/run/live/medium' squashfs/etc/apt/sources.list"
unmountSquashfs
}
function test_preexisting_package_inclusion_chroot_live() {
# Why this package?
# - It has only a few dependencies
# - It is not present in the small image
echo "hwdata" > config/package-lists/config-package-lists-chroot-live.list.chroot_live
build_image
assertFalse "Main package is not installed (install)" "grep -q '^hwdata' chroot.packages.install"
assertFalse "Dependency package is not installed (install)" "grep -q '^pci.ids' chroot.packages.install"
assertTrue "Main package is installed (live)" "grep -q '^hwdata' chroot.packages.live"
assertTrue "Dependency package is installed (live)" "grep -q '^pci.ids' chroot.packages.live"
mountSquashfs
assertTrue "Main package will be removed after installation" "grep -q '^hwdata' iso/live/filesystem.packages-remove"
assertTrue "Dependency package will be removed after installation" "grep -q '^pci\.ids' iso/live/filesystem.packages-remove"
assertFalse "No package pool should be generated" "[ -e iso/pool ]"
assertFalse "Package pool is not listed in /etc/apt/sources.list" "grep -q 'file:/run/live/medium' squashfs/etc/apt/sources.list"
unmountSquashfs
}
# Effectively is a duplicate of test_preexisting_package_inclusion_chroot
function test_preexisting_package_inclusion_chroot_install() {
# Why this package?
# - It has only a few dependencies
# - It is not present in the small image
echo "hwdata" > config/package-lists/config-package-lists-chroot-live.list.chroot_install
build_image
assertTrue "Main package is installed (install)" "grep -q '^hwdata' chroot.packages.install"
assertTrue "Dependency package is installed (install)" "grep -q '^pci.ids' chroot.packages.install"
assertTrue "Main package is installed (live)" "grep -q '^hwdata' chroot.packages.live"
assertTrue "Dependency package is installed (live)" "grep -q '^pci.ids' chroot.packages.live"
mountSquashfs
assertFalse "Main package stays after installation" "grep -q '^hwdata' iso/live/filesystem.packages-remove"
assertFalse "Dependency package stays after installation" "grep -q '^pci\.ids' iso/live/filesystem.packages-remove"
assertFalse "No package pool should be generated" "[ -e iso/pool ]"
assertFalse "Package pool is not listed in /etc/apt/sources.list" "grep -q 'file:/run/live/medium' squashfs/etc/apt/sources.list"
unmountSquashfs
}
function test_preexisting_package_inclusion_unspecified_chroot_or_binary() {
# Why this package?
# - It has only a few dependencies
# - It is not present in the small image
echo "hwdata" > config/package-lists/config-package-lists-chroot.list
build_image
assertTrue "Main package is installed (install)" "grep -q '^hwdata' chroot.packages.install"
assertTrue "Dependency package is installed (install)" "grep -q '^pci.ids' chroot.packages.install"
assertTrue "Main package is installed (live)" "grep -q '^hwdata' chroot.packages.live"
assertTrue "Dependency package is installed (live)" "grep -q '^pci.ids' chroot.packages.live"
mountSquashfs
assertFalse "Main package stays after installation" "grep -q '^hwdata' iso/live/filesystem.packages-remove"
assertFalse "Dependency package stays after installation" "grep -q '^pci\.ids' iso/live/filesystem.packages-remove"
assertTrue "Main package should be in the pool" "[ -e iso/pool/main/h/hwdata/hwdata_*_all.deb ]"
assertTrue "Dependency package should be in the pool" "[ -e iso/pool/main/p/pci.ids/pci.ids_*_all.deb ]"
assertTrue "Package pool is listed in /etc/apt/sources.list" "grep -q 'file:/run/live/medium' squashfs/etc/apt/sources.list"
assertTrue "Sources list meta info should be present: Release" "[ -e squashfs/var/lib/apt/lists/_run_live_medium_dists_unstable_Release ]"
assertTrue "Sources list meta info should be present: Packages" "[ -e squashfs/var/lib/apt/lists/_run_live_medium_dists_unstable_main_binary-amd64_Packages ]"
unmountSquashfs
}
function test_preexisting_package_inclusion_binary() {
# Why this package?
# - It has only a few dependencies
# - It is not present in the small image
echo "hwdata" > config/package-lists/config-package-lists-chroot.list.binary
build_image
assertFalse "Main package is not installed (install)" "grep -q '^hwdata' chroot.packages.install"
assertFalse "Dependency package is not installed (install)" "grep -q '^pci.ids' chroot.packages.install"
assertFalse "Main package is not installed (live)" "grep -q '^hwdata' chroot.packages.live"
assertFalse "Dependency package is not installed (live)" "grep -q '^pci.ids' chroot.packages.live"
mountSquashfs
assertFalse "Main package stays after installation" "grep -q '^hwdata' iso/live/filesystem.packages-remove"
assertFalse "Dependency package stays after installation" "grep -q '^pci\.ids' iso/live/filesystem.packages-remove"
assertTrue "Main package should be in the pool" "[ -e iso/pool/main/h/hwdata/hwdata_*_all.deb ]"
assertTrue "Dependency package should be in the pool" "[ -e iso/pool/main/p/pci.ids/pci.ids_*_all.deb ]"
assertTrue "Package pool is listed in /etc/apt/sources.list" "grep -q 'file:/run/live/medium' squashfs/etc/apt/sources.list"
assertTrue "Sources list meta info should be present: Release" "[ -e squashfs/var/lib/apt/lists/_run_live_medium_dists_unstable_Release ]"
assertTrue "Sources list meta info should be present: Packages" "[ -e squashfs/var/lib/apt/lists/_run_live_medium_dists_unstable_main_binary-amd64_Packages ]"
unmountSquashfs
}
# Untested:
# 8.2.5 Generated package lists
# 8.2.6 Using conditionals inside package lists
function test_direct_inclusion_of_deb_unspecified_chroot_or_binary() {
create_packages config-packages
cp live-testpackage-config-packages-main_1.0_all.deb config/packages
@ -155,8 +265,8 @@ function test_direct_inclusion_of_deb_binary() {
assertTrue "Main package should be in the pool" "[ -e iso/pool/main/l/live-testpackage-config-packages-binary-main/live-testpackage-config-packages-binary-main_1.0_all.deb ]"
assertTrue "Dependency package should be in the pool" "[ -e iso/pool/main/l/live-testpackage-config-packages-binary-dependency/live-testpackage-config-packages-binary-dependency_1.0_all.deb ]"
assertTrue "Package pool is listed in /etc/apt/sources.list" "grep -q 'file:/run/live/medium' squashfs/etc/apt/sources.list"
assertTrue "Sources list meta info should be present" "[ -e squashfs/var/lib/apt/lists/_run_live_medium_dists_unstable_Release ]"
assertTrue "Sources list meta info should be present" "[ -e squashfs/var/lib/apt/lists/_run_live_medium_dists_unstable_main_binary-amd64_Packages ]"
assertTrue "Sources list meta info should be present: Release" "[ -e squashfs/var/lib/apt/lists/_run_live_medium_dists_unstable_Release ]"
assertTrue "Sources list meta info should be present: Packages" "[ -e squashfs/var/lib/apt/lists/_run_live_medium_dists_unstable_main_binary-amd64_Packages ]"
unmountSquashfs
}
@ -314,8 +424,8 @@ EOF
assertTrue "Main package should be in the pool" "[ -e iso/pool/main/l/live-testpackage-config-archives-list-binary-main/live-testpackage-config-archives-list-binary-main_1.0_all.deb ]"
assertTrue "Dependency package should be in the pool" "[ -e iso/pool/main/l/live-testpackage-config-archives-list-binary-dependency/live-testpackage-config-archives-list-binary-dependency_1.0_all.deb ]"
assertTrue "Package pool is listed in /etc/apt/sources.list" "grep -q 'file:/run/live/medium' squashfs/etc/apt/sources.list"
assertTrue "Sources list meta info should be present" "[ -e squashfs/var/lib/apt/lists/_run_live_medium_dists_unstable_Release ]"
assertTrue "Sources list meta info should be present" "[ -e squashfs/var/lib/apt/lists/_run_live_medium_dists_unstable_main_binary-amd64_Packages ]"
assertTrue "Sources list meta info should be present: Release" "[ -e squashfs/var/lib/apt/lists/_run_live_medium_dists_unstable_Release ]"
assertTrue "Sources list meta info should be present: Packages" "[ -e squashfs/var/lib/apt/lists/_run_live_medium_dists_unstable_main_binary-amd64_Packages ]"
unmountSquashfs
}
@ -351,8 +461,8 @@ function test_derivatives() {
# Rebuild the configuration, as many mirror settings depend on eachother
#lb clean --purge
#rm -fr config
# Slight speedup: --zsync, --firmware-chroot
#lb config --distribution unstable --zsync false --firmware-chroot false
# Slight speedup: --zsync, --firmware-chroot, --cache
#lb config --distribution unstable --zsync false --firmware-chroot false --cache false
# Let's not test --parent-distribution-chroot at the moment:
# --apt-secure false --parent-mirror-chroot file://localhost$(pwd)/testrepository --parent-distribution-chroot nondebian --parent-archive-areas mymain --mirror-chroot http://deb.debian.org/debian --distribution-chroot debian --archive-areas main --parent-mirror-bootstrap file://localhost$(pwd)/testrepository
# --apt-secure false --mirror-chroot file://localhost$(pwd)/testrepository-mirror-chroot --distribution-chroot nondebian --archive-areas mymain --parent-mirror-chroot http://deb.debian.org/debian --parent-distribution-chroot unstable --parent-archive-areas main
@ -363,4 +473,7 @@ function test_derivatives() {
#unmountSquashfs
}
. shunit2
SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(date --utc '+%s')}"
ISO8601_TIMESTAMP=$(date --utc -d@${SOURCE_DATE_EPOCH} +%Y-%m-%dT%H:%M:%SZ)
. shunit2 2> logfile_${ISO8601_TIMESTAMP}.stderr | tee logfile_${ISO8601_TIMESTAMP}.stdout
egrep "ASSERT|FAILED|OK|shunit2|test_|SHA256" logfile_${ISO8601_TIMESTAMP}.stdout | tee logfile_${ISO8601_TIMESTAMP}.summary