Rebuild a reproducible ISO image.
The steps were initially published at https://wiki.debian.org/ReproducibleInstalls/LiveImages
This commit is contained in:
Roland Clobus
parent
c2263f1748
commit
a9d367d406
|
|
@ -1 +1,23 @@
|
||||||
coverage
|
coverage
|
||||||
|
|
||||||
|
# rebuild related folders
|
||||||
|
live-build
|
||||||
|
debian-installer
|
||||||
|
.build
|
||||||
|
binary
|
||||||
|
cache
|
||||||
|
chroot
|
||||||
|
config
|
||||||
|
|
||||||
|
# rebuild related files
|
||||||
|
binary.modified_timestamps
|
||||||
|
chroot.files
|
||||||
|
chroot.packages.install
|
||||||
|
chroot.packages.live
|
||||||
|
lb_output.txt
|
||||||
|
live-image-amd64.contents
|
||||||
|
live-image-amd64.files
|
||||||
|
live-image-amd64.hybrid.iso
|
||||||
|
live-image-amd64.hybrid.iso.zsync
|
||||||
|
live-image-amd64.packages
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,289 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# Rebuild an ISO image for a given timestamp
|
||||||
|
#
|
||||||
|
# Copyright 2021-2022 Holger Levsen <holger@layer-acht.org>
|
||||||
|
# Copyright 2021-2022 Roland Clobus <rclobus@rclobus.nl>
|
||||||
|
# released under the GPLv2
|
||||||
|
|
||||||
|
# Command line arguments:
|
||||||
|
# 1) Image type
|
||||||
|
# 2) Debian version
|
||||||
|
# 3) [optional] Timestamp (format: YYYYMMDD'T'HHMMSS'Z')
|
||||||
|
|
||||||
|
# Environment variables:
|
||||||
|
# http_proxy: The proxy that is used by live-build and wget
|
||||||
|
# https_proxy: The proxy that is used by git
|
||||||
|
# SNAPSHOT_TIMESTAMP: The timestamp to rebuild (format: YYYYMMDD'T'HHMMSS'Z')
|
||||||
|
|
||||||
|
# Required entries in the sudoers file:
|
||||||
|
# Defaults env_keep += "SOURCE_DATE_EPOCH"
|
||||||
|
# Defaults env_keep += "LIVE_BUILD"
|
||||||
|
# thisuser ALL=(root) NOPASSWD: /usr/bin/lb build
|
||||||
|
# thisuser ALL=(root) NOPASSWD: /usr/bin/lb clean --purge
|
||||||
|
|
||||||
|
# Coding convention: enforced by 'shfmt'
|
||||||
|
|
||||||
|
DEBUG=false
|
||||||
|
|
||||||
|
set -e
|
||||||
|
set -o pipefail # see eg http://petereisentraut.blogspot.com/2010/11/pipefail.html
|
||||||
|
|
||||||
|
output_echo() {
|
||||||
|
set +x
|
||||||
|
echo "###########################################################################################"
|
||||||
|
echo
|
||||||
|
echo -e "$(date -u) - $1"
|
||||||
|
echo
|
||||||
|
if $DEBUG; then
|
||||||
|
set -x
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
cleanup() {
|
||||||
|
output_echo "Generating summary.txt $1"
|
||||||
|
cat <<EOF >summary.txt
|
||||||
|
Configuration: ${CONFIGURATION}
|
||||||
|
Debian version: ${DEBIAN_VERSION}
|
||||||
|
Use latest snapshot: ${BUILD_LATEST}
|
||||||
|
Snapshot timestamp: ${SNAPSHOT_TIMESTAMP}
|
||||||
|
Snapshot epoch: ${SOURCE_DATE_EPOCH}
|
||||||
|
Live-build override: ${LIVE_BUILD_OVERRIDE}
|
||||||
|
Live-build path: ${LIVE_BUILD}
|
||||||
|
Build result: ${BUILD_RESULT}
|
||||||
|
Alternative timestamp: ${PROPOSED_SNAPSHOT_TIMESTAMP}
|
||||||
|
Checksum: ${SHA256SUM}
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
prepare_installer_cache() {
|
||||||
|
local SRC_FILENAME=$1
|
||||||
|
local DEST_FILENAME="$(pwd)/cache/installer_debian-installer/$(echo "${MIRROR}/dists/${DEBIAN_VERSION}/main/installer-amd64/current/images/${SRC_FILENAME}" | sed 's|/|_|g')"
|
||||||
|
|
||||||
|
mkdir -p $(dirname ${DEST_FILENAME})
|
||||||
|
cp -a ${DEBIAN_INSTALLER}/build/dest/${SRC_FILENAME} ${DEST_FILENAME}
|
||||||
|
}
|
||||||
|
|
||||||
|
parse_commandline_arguments() {
|
||||||
|
# Argument 1 = image type
|
||||||
|
case $1 in
|
||||||
|
"smallest-build")
|
||||||
|
export INSTALLER="none"
|
||||||
|
export PACKAGES=""
|
||||||
|
;;
|
||||||
|
"cinnamon")
|
||||||
|
export INSTALLER="live"
|
||||||
|
export PACKAGES="live-task-cinnamon"
|
||||||
|
;;
|
||||||
|
"gnome")
|
||||||
|
export INSTALLER="live"
|
||||||
|
export PACKAGES="live-task-gnome"
|
||||||
|
;;
|
||||||
|
"kde")
|
||||||
|
export INSTALLER="live"
|
||||||
|
export PACKAGES="live-task-kde"
|
||||||
|
;;
|
||||||
|
"lxde")
|
||||||
|
export INSTALLER="live"
|
||||||
|
export PACKAGES="live-task-lxde"
|
||||||
|
;;
|
||||||
|
"lxqt")
|
||||||
|
export INSTALLER="live"
|
||||||
|
export PACKAGES="live-task-lxqt"
|
||||||
|
;;
|
||||||
|
"mate")
|
||||||
|
export INSTALLER="live"
|
||||||
|
export PACKAGES="live-task-mate"
|
||||||
|
;;
|
||||||
|
"standard")
|
||||||
|
export INSTALLER="live"
|
||||||
|
export PACKAGES="live-task-standard"
|
||||||
|
;;
|
||||||
|
"xfce")
|
||||||
|
export INSTALLER="live"
|
||||||
|
export PACKAGES="live-task-xfce"
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
output_echo "Error: Bad argument 1, image type: $1"
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
export CONFIGURATION="$1"
|
||||||
|
|
||||||
|
# Argument 2 = Debian version
|
||||||
|
# Use 'stable', 'testing' or 'unstable' or code names like 'sid'
|
||||||
|
if [ -z "$2" ]; then
|
||||||
|
output_echo "Error: Bad argument 2, Debian version: it is empty"
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
export DEBIAN_VERSION="$2"
|
||||||
|
|
||||||
|
# Argument 3 = optional timestamp
|
||||||
|
export BUILD_LATEST=1
|
||||||
|
if [ ! -z "$3" ]; then
|
||||||
|
export SNAPSHOT_TIMESTAMP=$3
|
||||||
|
BUILD_LATEST=0
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
#
|
||||||
|
# main: follow https://wiki.debian.org/ReproducibleInstalls/LiveImages
|
||||||
|
#
|
||||||
|
|
||||||
|
# Cleanup if something goes wrong
|
||||||
|
trap cleanup INT TERM EXIT
|
||||||
|
|
||||||
|
parse_commandline_arguments "$@"
|
||||||
|
|
||||||
|
if $DEBUG; then
|
||||||
|
export WGET_OPTIONS=
|
||||||
|
export GIT_OPTIONS=
|
||||||
|
else
|
||||||
|
export WGET_OPTIONS=--quiet
|
||||||
|
export GIT_OPTIONS=--quiet
|
||||||
|
fi
|
||||||
|
|
||||||
|
# No log required
|
||||||
|
WGET_OPTIONS="${WGET_OPTIONS} --output-file /dev/null --timestamping"
|
||||||
|
|
||||||
|
if [ ! -z "${LIVE_BUILD}" ]; then
|
||||||
|
export LIVE_BUILD_OVERRIDE=1
|
||||||
|
else
|
||||||
|
export LIVE_BUILD_OVERRIDE=0
|
||||||
|
export LIVE_BUILD=${PWD}/live-build
|
||||||
|
fi
|
||||||
|
export DEBIAN_INSTALLER=${PWD}/debian-installer
|
||||||
|
|
||||||
|
# Use a fresh git clone
|
||||||
|
if [ ! -d ${LIVE_BUILD} -a ${LIVE_BUILD_OVERRIDE} -eq 0 ]; then
|
||||||
|
git clone https://salsa.debian.org/live-team/live-build.git ${LIVE_BUILD} --single-branch --no-tags
|
||||||
|
fi
|
||||||
|
if [ ! -d ${DEBIAN_INSTALLER} -a "${INSTALLER}" == "live" ]; then
|
||||||
|
git clone https://salsa.debian.org/installer-team/debian-installer.git ${DEBIAN_INSTALLER} --single-branch --no-tags
|
||||||
|
fi
|
||||||
|
|
||||||
|
export LB_OUTPUT=lb_output.txt
|
||||||
|
rm -f ${LB_OUTPUT}
|
||||||
|
|
||||||
|
if [ ${BUILD_LATEST} -eq 1 ]; then
|
||||||
|
# Use the timestamp of the latest mirror snapshot
|
||||||
|
wget ${WGET_OPTIONS} http://snapshot.notset.fr/mr/timestamp/debian/latest --output-document latest
|
||||||
|
#
|
||||||
|
# Extract the timestamp from the JSON file
|
||||||
|
#
|
||||||
|
# Input:
|
||||||
|
# {
|
||||||
|
# "_api": "0.3",
|
||||||
|
# "_comment": "notset",
|
||||||
|
# "result": "20210828T083909Z"
|
||||||
|
# }
|
||||||
|
# Output:
|
||||||
|
# 20210828T083909Z
|
||||||
|
#
|
||||||
|
export SNAPSHOT_TIMESTAMP=$(cat latest | awk '/"result":/ { split($0, a, "\""); print a[4] }')
|
||||||
|
rm latest
|
||||||
|
fi
|
||||||
|
# Convert SNAPSHOT_TIMESTAMP to Unix time (insert suitable formatting first)
|
||||||
|
export SOURCE_DATE_EPOCH=$(date -d $(echo ${SNAPSHOT_TIMESTAMP} | awk '{ printf "%s-%s-%sT%s:%s:%sZ", substr($0,1,4), substr($0,5,2), substr($0,7,2), substr($0,10,2), substr($0,12,2), substr($0,14,2) }') +%s)
|
||||||
|
export MIRROR=http://snapshot.notset.fr/archive/debian/${SNAPSHOT_TIMESTAMP}
|
||||||
|
output_echo "Info: using the snapshot from ${SOURCE_DATE_EPOCH} (${SNAPSHOT_TIMESTAMP})"
|
||||||
|
|
||||||
|
# Use the code from the actual timestamp
|
||||||
|
# Report the versions that were actually used
|
||||||
|
if [ ${LIVE_BUILD_OVERRIDE} -eq 0 ]; then
|
||||||
|
pushd ${LIVE_BUILD} >/dev/null
|
||||||
|
git pull ${GIT_OPTIONS}
|
||||||
|
git checkout $(git rev-list -n 1 --min-age=${SOURCE_DATE_EPOCH} HEAD) ${GIT_OPTIONS}
|
||||||
|
git clean -Xdf ${GIT_OPTIONS}
|
||||||
|
output_echo "Info: using live-build from git version $(git log -n 1 --pretty=format:%H_%aI)"
|
||||||
|
popd >/dev/null
|
||||||
|
else
|
||||||
|
output_echo "Info: using local live-build: $(lb --version)"
|
||||||
|
fi
|
||||||
|
if [ "${INSTALLER}" == "live" ]; then
|
||||||
|
pushd ${DEBIAN_INSTALLER} >/dev/null
|
||||||
|
git pull ${GIT_OPTIONS}
|
||||||
|
git checkout $(git rev-list -n 1 --min-age=${SOURCE_DATE_EPOCH} HEAD) ${GIT_OPTIONS}
|
||||||
|
git clean -Xdf ${GIT_OPTIONS}
|
||||||
|
# Specific patch to prevent /etc/apt/apt.conf.d of the host from influencing the dependency generator
|
||||||
|
sed -i -e 's|LANG=C apt-cache|LANG=C APT_CONFIG=./apt.udeb/apt.conf apt-cache|' build/util/pkg-list
|
||||||
|
output_echo "Info: using debian-installer from git version $(git log -n 1 --pretty=format:%H_%aI)"
|
||||||
|
popd >/dev/null
|
||||||
|
fi
|
||||||
|
|
||||||
|
# If the configuration folder already exists, re-create from scratch
|
||||||
|
if [ -d config ]; then
|
||||||
|
sudo lb clean --purge
|
||||||
|
rm -fr config
|
||||||
|
rm -fr .build
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "${INSTALLER}" == "live" ]; then
|
||||||
|
# Create a snapshot for the debian-installer
|
||||||
|
pushd ${DEBIAN_INSTALLER} >/dev/null
|
||||||
|
cd build
|
||||||
|
# MIRROR -> our snapshot URL, with disabled expiration
|
||||||
|
# TARGETS -> only these targets are required
|
||||||
|
MIRROR="[check-valid-until=no] ${MIRROR}" TARGETS="build_cdrom_gtk build_cdrom_isolinux" bash ./daily-build build-only
|
||||||
|
popd >/dev/null
|
||||||
|
|
||||||
|
# Copy the fresh installer to the cache. This will be used instead of the content of the online archive
|
||||||
|
prepare_installer_cache cdrom/vmlinuz
|
||||||
|
prepare_installer_cache cdrom/initrd.gz
|
||||||
|
prepare_installer_cache cdrom/gtk/vmlinuz
|
||||||
|
prepare_installer_cache cdrom/gtk/initrd.gz
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Configuration for the live image:
|
||||||
|
# - For /etc/apt/sources.list: Use the mirror from ${MIRROR}, no security, no updates
|
||||||
|
# - Don't cache the downloaded content
|
||||||
|
# - To reduce some network traffic a proxy is implicitly used
|
||||||
|
output_echo "Running lb config."
|
||||||
|
lb config \
|
||||||
|
--parent-mirror-bootstrap ${MIRROR} \
|
||||||
|
--parent-mirror-binary ${MIRROR} \
|
||||||
|
--security false \
|
||||||
|
--updates false \
|
||||||
|
--distribution ${DEBIAN_VERSION} \
|
||||||
|
--debian-installer ${INSTALLER} \
|
||||||
|
--cache-packages false \
|
||||||
|
2>&1 | tee $LB_OUTPUT
|
||||||
|
|
||||||
|
# Insider knowledge of live-build:
|
||||||
|
# Add '-o Acquire::Check-Valid-Until=false', to allow for rebuilds of older timestamps
|
||||||
|
sed -i -e '/^APT_OPTIONS=/s/--yes/--yes -o Acquire::Check-Valid-Until=false/' config/common
|
||||||
|
|
||||||
|
if [ ! -z "${PACKAGES}" ]; then
|
||||||
|
echo "${PACKAGES}" >config/package-lists/desktop.list.chroot
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Add additional hooks, that work around known issues regarding reproducibility
|
||||||
|
cp -a ${LIVE_BUILD}/examples/hooks/reproducible/* config/hooks/normal
|
||||||
|
|
||||||
|
# Build the image
|
||||||
|
output_echo "Running lb build."
|
||||||
|
|
||||||
|
set +e # We are interested in the result of 'lb build', so do not fail on errors
|
||||||
|
sudo lb build | tee -a $LB_OUTPUT
|
||||||
|
export BUILD_RESULT=$?
|
||||||
|
set -e
|
||||||
|
if [ ${BUILD_RESULT} -ne 0 ]; then
|
||||||
|
# Find the snapshot that matches 1 second before the current snapshot
|
||||||
|
wget ${WGET_OPTIONS} http://snapshot.notset.fr/mr/timestamp/debian/$(date --utc -d @$((${SOURCE_DATE_EPOCH} - 1)) +%Y%m%dT%H%M%SZ) --output-document but_latest
|
||||||
|
export PROPOSED_SNAPSHOT_TIMESTAMP=$(cat but_latest | awk '/"result":/ { split($0, a, "\""); print a[4] }')
|
||||||
|
rm but_latest
|
||||||
|
|
||||||
|
output_echo "Warning: lb build failed with ${BUILD_RESULT}. The latest snapshot might not be complete (yet). Try re-running the script with SNAPSHOT_TIMESTAMP=${PROPOSED_SNAPSHOT_TIMESTAMP}."
|
||||||
|
# Occasionally the snapshot is not complete, you could use the previous snapshot instead of giving up
|
||||||
|
exit 99
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Calculate the checksum
|
||||||
|
export SHA256SUM=$(sha256sum live-image-amd64.hybrid.iso | cut -f 1 -d " ")
|
||||||
|
|
||||||
|
cleanup success
|
||||||
|
# Turn off the trap
|
||||||
|
trap - INT TERM EXIT
|
||||||
|
|
||||||
|
# We reached the end, return with PASS
|
||||||
|
exit 0
|
||||||
Loading…
Reference in New Issue