diff --git a/test/.gitignore b/test/.gitignore index 4ebc8aea5..79379229d 100644 --- a/test/.gitignore +++ b/test/.gitignore @@ -1 +1,23 @@ coverage + +# rebuild related folders +live-build +debian-installer +.build +binary +cache +chroot +config + +# rebuild related files +binary.modified_timestamps +chroot.files +chroot.packages.install +chroot.packages.live +lb_output.txt +live-image-amd64.contents +live-image-amd64.files +live-image-amd64.hybrid.iso +live-image-amd64.hybrid.iso.zsync +live-image-amd64.packages + diff --git a/test/rebuild.sh b/test/rebuild.sh new file mode 100755 index 000000000..667440055 --- /dev/null +++ b/test/rebuild.sh @@ -0,0 +1,289 @@ +#!/bin/bash + +# Rebuild an ISO image for a given timestamp +# +# Copyright 2021-2022 Holger Levsen +# Copyright 2021-2022 Roland Clobus +# released under the GPLv2 + +# Command line arguments: +# 1) Image type +# 2) Debian version +# 3) [optional] Timestamp (format: YYYYMMDD'T'HHMMSS'Z') + +# Environment variables: +# http_proxy: The proxy that is used by live-build and wget +# https_proxy: The proxy that is used by git +# SNAPSHOT_TIMESTAMP: The timestamp to rebuild (format: YYYYMMDD'T'HHMMSS'Z') + +# Required entries in the sudoers file: +# Defaults env_keep += "SOURCE_DATE_EPOCH" +# Defaults env_keep += "LIVE_BUILD" +# thisuser ALL=(root) NOPASSWD: /usr/bin/lb build +# thisuser ALL=(root) NOPASSWD: /usr/bin/lb clean --purge + +# Coding convention: enforced by 'shfmt' + +DEBUG=false + +set -e +set -o pipefail # see eg http://petereisentraut.blogspot.com/2010/11/pipefail.html + +output_echo() { + set +x + echo "###########################################################################################" + echo + echo -e "$(date -u) - $1" + echo + if $DEBUG; then + set -x + fi +} + +cleanup() { + output_echo "Generating summary.txt $1" + cat <summary.txt +Configuration: ${CONFIGURATION} +Debian version: ${DEBIAN_VERSION} +Use latest snapshot: ${BUILD_LATEST} +Snapshot timestamp: ${SNAPSHOT_TIMESTAMP} +Snapshot epoch: ${SOURCE_DATE_EPOCH} +Live-build override: ${LIVE_BUILD_OVERRIDE} +Live-build path: ${LIVE_BUILD} +Build result: ${BUILD_RESULT} +Alternative timestamp: ${PROPOSED_SNAPSHOT_TIMESTAMP} +Checksum: ${SHA256SUM} +EOF +} + +prepare_installer_cache() { + local SRC_FILENAME=$1 + local DEST_FILENAME="$(pwd)/cache/installer_debian-installer/$(echo "${MIRROR}/dists/${DEBIAN_VERSION}/main/installer-amd64/current/images/${SRC_FILENAME}" | sed 's|/|_|g')" + + mkdir -p $(dirname ${DEST_FILENAME}) + cp -a ${DEBIAN_INSTALLER}/build/dest/${SRC_FILENAME} ${DEST_FILENAME} +} + +parse_commandline_arguments() { + # Argument 1 = image type + case $1 in + "smallest-build") + export INSTALLER="none" + export PACKAGES="" + ;; + "cinnamon") + export INSTALLER="live" + export PACKAGES="live-task-cinnamon" + ;; + "gnome") + export INSTALLER="live" + export PACKAGES="live-task-gnome" + ;; + "kde") + export INSTALLER="live" + export PACKAGES="live-task-kde" + ;; + "lxde") + export INSTALLER="live" + export PACKAGES="live-task-lxde" + ;; + "lxqt") + export INSTALLER="live" + export PACKAGES="live-task-lxqt" + ;; + "mate") + export INSTALLER="live" + export PACKAGES="live-task-mate" + ;; + "standard") + export INSTALLER="live" + export PACKAGES="live-task-standard" + ;; + "xfce") + export INSTALLER="live" + export PACKAGES="live-task-xfce" + ;; + *) + output_echo "Error: Bad argument 1, image type: $1" + exit 1 + ;; + esac + export CONFIGURATION="$1" + + # Argument 2 = Debian version + # Use 'stable', 'testing' or 'unstable' or code names like 'sid' + if [ -z "$2" ]; then + output_echo "Error: Bad argument 2, Debian version: it is empty" + exit 2 + fi + export DEBIAN_VERSION="$2" + + # Argument 3 = optional timestamp + export BUILD_LATEST=1 + if [ ! -z "$3" ]; then + export SNAPSHOT_TIMESTAMP=$3 + BUILD_LATEST=0 + fi +} + +# +# main: follow https://wiki.debian.org/ReproducibleInstalls/LiveImages +# + +# Cleanup if something goes wrong +trap cleanup INT TERM EXIT + +parse_commandline_arguments "$@" + +if $DEBUG; then + export WGET_OPTIONS= + export GIT_OPTIONS= +else + export WGET_OPTIONS=--quiet + export GIT_OPTIONS=--quiet +fi + +# No log required +WGET_OPTIONS="${WGET_OPTIONS} --output-file /dev/null --timestamping" + +if [ ! -z "${LIVE_BUILD}" ]; then + export LIVE_BUILD_OVERRIDE=1 +else + export LIVE_BUILD_OVERRIDE=0 + export LIVE_BUILD=${PWD}/live-build +fi +export DEBIAN_INSTALLER=${PWD}/debian-installer + +# Use a fresh git clone +if [ ! -d ${LIVE_BUILD} -a ${LIVE_BUILD_OVERRIDE} -eq 0 ]; then + git clone https://salsa.debian.org/live-team/live-build.git ${LIVE_BUILD} --single-branch --no-tags +fi +if [ ! -d ${DEBIAN_INSTALLER} -a "${INSTALLER}" == "live" ]; then + git clone https://salsa.debian.org/installer-team/debian-installer.git ${DEBIAN_INSTALLER} --single-branch --no-tags +fi + +export LB_OUTPUT=lb_output.txt +rm -f ${LB_OUTPUT} + +if [ ${BUILD_LATEST} -eq 1 ]; then + # Use the timestamp of the latest mirror snapshot + wget ${WGET_OPTIONS} http://snapshot.notset.fr/mr/timestamp/debian/latest --output-document latest + # + # Extract the timestamp from the JSON file + # + # Input: + # { + # "_api": "0.3", + # "_comment": "notset", + # "result": "20210828T083909Z" + # } + # Output: + # 20210828T083909Z + # + export SNAPSHOT_TIMESTAMP=$(cat latest | awk '/"result":/ { split($0, a, "\""); print a[4] }') + rm latest +fi +# Convert SNAPSHOT_TIMESTAMP to Unix time (insert suitable formatting first) +export SOURCE_DATE_EPOCH=$(date -d $(echo ${SNAPSHOT_TIMESTAMP} | awk '{ printf "%s-%s-%sT%s:%s:%sZ", substr($0,1,4), substr($0,5,2), substr($0,7,2), substr($0,10,2), substr($0,12,2), substr($0,14,2) }') +%s) +export MIRROR=http://snapshot.notset.fr/archive/debian/${SNAPSHOT_TIMESTAMP} +output_echo "Info: using the snapshot from ${SOURCE_DATE_EPOCH} (${SNAPSHOT_TIMESTAMP})" + +# Use the code from the actual timestamp +# Report the versions that were actually used +if [ ${LIVE_BUILD_OVERRIDE} -eq 0 ]; then + pushd ${LIVE_BUILD} >/dev/null + git pull ${GIT_OPTIONS} + git checkout $(git rev-list -n 1 --min-age=${SOURCE_DATE_EPOCH} HEAD) ${GIT_OPTIONS} + git clean -Xdf ${GIT_OPTIONS} + output_echo "Info: using live-build from git version $(git log -n 1 --pretty=format:%H_%aI)" + popd >/dev/null +else + output_echo "Info: using local live-build: $(lb --version)" +fi +if [ "${INSTALLER}" == "live" ]; then + pushd ${DEBIAN_INSTALLER} >/dev/null + git pull ${GIT_OPTIONS} + git checkout $(git rev-list -n 1 --min-age=${SOURCE_DATE_EPOCH} HEAD) ${GIT_OPTIONS} + git clean -Xdf ${GIT_OPTIONS} + # Specific patch to prevent /etc/apt/apt.conf.d of the host from influencing the dependency generator + sed -i -e 's|LANG=C apt-cache|LANG=C APT_CONFIG=./apt.udeb/apt.conf apt-cache|' build/util/pkg-list + output_echo "Info: using debian-installer from git version $(git log -n 1 --pretty=format:%H_%aI)" + popd >/dev/null +fi + +# If the configuration folder already exists, re-create from scratch +if [ -d config ]; then + sudo lb clean --purge + rm -fr config + rm -fr .build +fi + +if [ "${INSTALLER}" == "live" ]; then + # Create a snapshot for the debian-installer + pushd ${DEBIAN_INSTALLER} >/dev/null + cd build + # MIRROR -> our snapshot URL, with disabled expiration + # TARGETS -> only these targets are required + MIRROR="[check-valid-until=no] ${MIRROR}" TARGETS="build_cdrom_gtk build_cdrom_isolinux" bash ./daily-build build-only + popd >/dev/null + + # Copy the fresh installer to the cache. This will be used instead of the content of the online archive + prepare_installer_cache cdrom/vmlinuz + prepare_installer_cache cdrom/initrd.gz + prepare_installer_cache cdrom/gtk/vmlinuz + prepare_installer_cache cdrom/gtk/initrd.gz +fi + +# Configuration for the live image: +# - For /etc/apt/sources.list: Use the mirror from ${MIRROR}, no security, no updates +# - Don't cache the downloaded content +# - To reduce some network traffic a proxy is implicitly used +output_echo "Running lb config." +lb config \ + --parent-mirror-bootstrap ${MIRROR} \ + --parent-mirror-binary ${MIRROR} \ + --security false \ + --updates false \ + --distribution ${DEBIAN_VERSION} \ + --debian-installer ${INSTALLER} \ + --cache-packages false \ + 2>&1 | tee $LB_OUTPUT + +# Insider knowledge of live-build: +# Add '-o Acquire::Check-Valid-Until=false', to allow for rebuilds of older timestamps +sed -i -e '/^APT_OPTIONS=/s/--yes/--yes -o Acquire::Check-Valid-Until=false/' config/common + +if [ ! -z "${PACKAGES}" ]; then + echo "${PACKAGES}" >config/package-lists/desktop.list.chroot +fi + +# Add additional hooks, that work around known issues regarding reproducibility +cp -a ${LIVE_BUILD}/examples/hooks/reproducible/* config/hooks/normal + +# Build the image +output_echo "Running lb build." + +set +e # We are interested in the result of 'lb build', so do not fail on errors +sudo lb build | tee -a $LB_OUTPUT +export BUILD_RESULT=$? +set -e +if [ ${BUILD_RESULT} -ne 0 ]; then + # Find the snapshot that matches 1 second before the current snapshot + wget ${WGET_OPTIONS} http://snapshot.notset.fr/mr/timestamp/debian/$(date --utc -d @$((${SOURCE_DATE_EPOCH} - 1)) +%Y%m%dT%H%M%SZ) --output-document but_latest + export PROPOSED_SNAPSHOT_TIMESTAMP=$(cat but_latest | awk '/"result":/ { split($0, a, "\""); print a[4] }') + rm but_latest + + output_echo "Warning: lb build failed with ${BUILD_RESULT}. The latest snapshot might not be complete (yet). Try re-running the script with SNAPSHOT_TIMESTAMP=${PROPOSED_SNAPSHOT_TIMESTAMP}." + # Occasionally the snapshot is not complete, you could use the previous snapshot instead of giving up + exit 99 +fi + +# Calculate the checksum +export SHA256SUM=$(sha256sum live-image-amd64.hybrid.iso | cut -f 1 -d " ") + +cleanup success +# Turn off the trap +trap - INT TERM EXIT + +# We reached the end, return with PASS +exit 0