From 081cca8e99e2bb6ee7706e45a06fa201d90dd2ff Mon Sep 17 00:00:00 2001 From: Steve McIntyre <93sam@debian.org> Date: Sun, 14 Nov 2010 15:13:48 +0000 Subject: [PATCH] extra scripts in use for regular daily and weekly builds --- contrib/CONF.sh | 305 +++++++++++++++++++++++++++++++ contrib/common.sh | 69 +++++++ contrib/generate_firmware_images | 39 ++++ contrib/iso_run | 93 ++++++++++ contrib/merge-sums | 40 ++++ contrib/mklist | 17 ++ contrib/mktorrent | 14 ++ contrib/report_build_error | 38 ++++ 8 files changed, 615 insertions(+) create mode 100644 contrib/CONF.sh create mode 100644 contrib/common.sh create mode 100755 contrib/generate_firmware_images create mode 100755 contrib/iso_run create mode 100755 contrib/merge-sums create mode 100755 contrib/mklist create mode 100755 contrib/mktorrent create mode 100755 contrib/report_build_error diff --git a/contrib/CONF.sh b/contrib/CONF.sh new file mode 100644 index 00000000..f71376e2 --- /dev/null +++ b/contrib/CONF.sh @@ -0,0 +1,305 @@ +# +# This file will have to be sourced where needed +# + +# Unset all optional variables first to start from a clean state +unset NONUS || true +unset FORCENONUSONCD1 || true +unset NONFREE || true +unset CONTRIB || true +unset EXTRANONFREE || true +unset LOCAL || true +unset LOCALDEBS || true +unset SECURED || true +unset SECURITY || true +unset BOOTDIR || true +unset BOOTDISKS || true +unset SYMLINK || true +unset COPYLINK || true +unset MKISOFS || true +unset MKISOFS_OPTS || true +unset ISOLINUX || true +unset EXCLUDE || true +unset SRCEXCLUDE || true +unset NORECOMMENDS || true +unset NOSUGGESTS || true +unset IMAGESUMS || true +unset JIGDOCMD || true +unset JIGDOTEMPLATEURL || true +#unset JIGDOFALLBACKURLS || true +unset JIGDOINCLUDEURLS || true +unset JIGDOSCRIPT || true +unset JIGDO_OPTS || true +#unset DEFBINSIZE || true +#unset DEFSRCSIZE || true +unset FASTSUMS || true +unset PUBLISH_URL || true +unset PUBLISH_NONUS_URL || true +unset PUBLISH_PATH || true +unset UDEB_INCLUDE || true +unset UDEB_EXCLUDE || true +unset BASE_INCLUDE || true +unset BASE_EXCLUDE || true +#unset INSTALLER_CD || true + + +# The debian-cd dir +# Where I am (hoping I'm in the debian-cd dir) +export BASEDIR=`pwd` + +# Building squeeze cd set ... +export CODENAME=squeeze + +if [ ! "$DI_CODENAME" ] +then + export DI_CODENAME=$CODENAME +fi + +# Version number, "2.2 r0", "2.2 r1" etc. +#export DEBVERSION="Lenny-DI-rc2" +export DEBVERSION="testing" + +# Official or non-official set. +# NOTE: THE "OFFICIAL" DESIGNATION IS ONLY ALLOWED FOR IMAGES AVAILABLE +# ON THE OFFICIAL DEBIAN CD WEBSITE http://cdimage.debian.org +#export OFFICIAL="Unofficial" +export OFFICIAL="Official Snapshot" +#export OFFICIAL="Official RC" +#export OFFICIAL="Official" + +# ... for arch +if [ ! "$ARCH" ] +then + export ARCH=`dpkg --print-installation-architecture` +fi + +# IMPORTANT : The 4 following paths must be on the same partition/device. +# If they aren't then you must set COPYLINK below to 1. This +# takes a lot of extra room to create the sandbox for the ISO +# images, however. Also, if you are using an NFS partition for +# some part of this, you must use this option. +# Paths to the mirrors +if [ "$MIRROR"x = ""x ] ; then + export MIRROR=/org/cdbuilder.debian.org/src/ftp/debian +fi + +NUM_ARCHES=`echo $ARCH | wc -w` +if [ "$NUM_ARCHES"x = "1"x ] ; then + OUTARCH=$ARCH +else + OUTARCH=multi-arch +fi + +# Path of the temporary directory +export TDIR=/org/cdbuilder.debian.org/src/deb-cd/tmp/"$INSTALLER_CD""$DI""$OUTARCH" + +# Path where the images will be written +if [ "$OUT"x = ""x ] ; then + export OUT=/org/cdbuilder.debian.org/dst/deb-cd/out/"$INSTALLER_CD""$DI""$OUTARCH" +fi + +# Where we keep the temporary apt stuff. +# This cannot reside on an NFS mount. +export APTTMP=$TDIR/apt + +# Do I want to have NONFREE merged in the CD set +# export NONFREE=1 + +# Do I want to have CONTRIB merged in the CD set +export CONTRIB=1 + +# Do I want to have NONFREE on a separate CD (the last CD of the CD set) +# WARNING: Don't use NONFREE and EXTRANONFREE at the same time ! +# export EXTRANONFREE=1 + +# If you have a $MIRROR/dists/$CODENAME/local/binary-$ARCH dir with +# local packages that you want to put on the CD set then +# uncomment the following line +# export LOCAL=1 + +# If your local packages are not under $MIRROR, but somewhere else, +# you can uncomment this line and edit to to point to a directory +# containing dists/$CODENAME/local/binary-$ARCH +# export LOCALDEBS=/home/joey/debian/va/debian + +# If you want a -secured tree with a copy of the signed +# Release.gpg and files listed by this Release file, then +# uncomment this line +# export SECURED=1 + +# Where to find the security patches. This directory should be the +# top directory of a security.debian.org mirror. +#export SECURITY="$TOPDIR"/debian/debian-security + +# Sparc only : bootdir (location of cd.b and second.b) +# export BOOTDIR=/boot + +# Symlink farmers should uncomment this line : +# export SYMLINK=1 + +# Use this to force copying the files instead of symlinking or hardlinking +# them. This is useful if your destination directories are on a different +# partition than your source files. +# export COPYLINK=1 + +# Options +#export MKISOFS="$BASEDIR/../mkisofs/usr/bin/mkisofs" +#export MKISOFS="$BASEDIR/../genisoimage/usr/bin/genisoimage" +export MKISOFS="$BASEDIR/../genisoimage" +#export MKISOFS_OPTS="-jigdo-template-compress bzip2 -r -checksum_algorithm_iso md5,sha1,sha256,sha512" +export MKISOFS_OPTS="-jigdo-template-compress bzip2 -r " #-checksum_algorithm_iso md5,sha1" +# export MKISOFS_OPTS="-r" #For normal users +# export MKISOFS_OPTS="-r -F ." #For symlink farmers + +# ISOLinux support for multiboot on CD1 for i386 +export ISOLINUX=1 + +# uncomment this to if you want to see more of what the Makefile is doing +export VERBOSE_MAKE=1 + +# The maximum size allowed for an individual package, in bytes; if +# larger than this, it will be excluded (and all dependents, of +# course) +#export MAX_PKG_SIZE=600000000 + +# uncoment this to make build_all.sh try to build a simple CD image if +# the proper official CD run does not work +#ATTEMPT_FALLBACK=yes + +# Set your disk size here in MB. Used in calculating package and +# source file layouts in build.sh and build_all.sh. Defaults are for +# CD-R, try ~4600 for DVD-R. +if [ ! "$DEFBINSIZE" ] ; then export DEFBINSIZE=642 ; fi +if [ ! "$DEFSRCSIZE" ] ; then export DEFSRCSIZE=642 ; fi + +if [ "$DISKTYPE"x = ""x ] ; then + DISKTYPE=CD +fi +export DISKTYPE + +# List of languages for which language tasks from tasksel should be +# included. See tasks/README.tasksel for further info. +export TASK_LANGLIST=tasksel_d-i.languages + +# We don't want certain packages to take up space on CD1... +#export EXCLUDE1=exclude +# ...but they are okay for other CDs (UNEXCLUDEx == will be included on CD x if not already covered) +#export UNEXCLUDE2=unexclude-CD2 +# Any packages listed in EXCLUDE but not in any UNEXCLUDE will be +# excluded completely. + +# We also exclude some source packages +#export SRCEXCLUDE=exclude-src + +# Set this if the recommended packages should be skipped when adding +# package on the CD. The default is 'false'. +export NORECOMMENDS=1 + +# Set this if the suggested packages should be skipped when adding +# package on the CD. The default is 'true'. +#export NOSUGGESTS=1 + +# Set to 1 to generate MD5 and SHA1 sums for generated images +export IMAGESUMS=1 + +# We may have to extract files from packages to put them onto the CD +# (e.g. bootloader files). If you make those packages (and their +# sources) available somewhere, list it here so that README.source +# can point to it +export ARCHIVE_EXTRACTED_SOURCES="http://cdimage.debian.org/cdimage/cd-sources/" + +# Produce iso/jigdo files: specify how many iso/jigdo files should be +# produced in your set. If not set or when the value is "ALL" they will +# be created for all images. One of the variables can be set to zero if +# either iso or jigdo files are not wanted, but not both. +# Replaces the old "DOJIGDO" setting with something much more flexible. +#export MAXISOS=0 +#export MAXJIGDOS=0 + +# Space-separated list of "include URLs" to add to the .jigdo file. +# The included files are used to provide an up-to-date list of Debian +# mirrors to the jigdo _GUI_application_ (_jigdo-lite_ doesn't support +# "[Include ...]"). +export JIGDOINCLUDEURLS="http://cdimage.debian.org/debian-cd/debian-servers.jigdo" +# +# $JIGDOTEMPLATEURL and $JIGDOINCLUDEURLS are passed to +# "tools/jigdo_header", which is used by default to generate the +# [Image] and [Servers] sections of the .jigdo file. You can provide +# your own script if you need the .jigdo file to contain different +# data. +#export JIGDOSCRIPT="myscript" + +# If set, use the md5sums from the main archive, rather than calculating +# them locally +export FASTSUMS=1 + +# A couple of things used only by publish_cds, so it can tweak the +# jigdo files, and knows where to put the results. +# You need to run publish_cds manually, it is not run by the Makefile. +#export PUBLISH_URL="http://cdimage.debian.org/jigdo-area" +#export PUBLISH_NONUS_URL="http://non-US.cdimage.debian.org/jigdo-area" +#export PUBLISH_PATH="/home/jigdo-area/" + +# Specify files and directories to *exclude* from jigdo processing. These +# files on each CD are expected to be different to those on the mirror, or +# are often subject to change. Any files matching entries in this list will +# simply be placed straight into the template file. +export JIGDO_EXCLUDE="'README*' /doc/ /md5sum.txt /.disk/ /pics/ 'Release*' 'Packages*' 'Sources*'" + +# Specify the minimum file size to consider for jigdo processing. Any files +# smaller than this will simply be placed straight into the template file. +export JIGDO_OPTS="-jigdo-min-file-size 0" + +for EXCL in $JIGDO_EXCLUDE +do + JIGDO_OPTS="$JIGDO_OPTS -jigdo-exclude $EXCL" +done + +export IGNORE_MISSING_BOOT_SCRIPT=1 + +# Where to find the boot disks +#export BOOTDISKS=$TOPDIR/ftp/skolelinux/boot-floppies + +# File with list of packages to include when fetching modules for the +# first stage installer (debian-installer). One package per line. +# Lines starting with '#' are comments. The package order is +# important, as the packages will be installed in the given order. +#export UDEB_INCLUDE="$BASEDIR"/data/$CODENAME/udeb_include + +# File with list of packages to exclude as above. +#export UDEB_EXCLUDE="$BASEDIR"/data/$CODENAME/udeb_exclude + +# File with list of packages to include when running debootstrap from +# the first stage installer (currently only supported in +# debian-installer). One package per line. Lines starting with '#' +# are comments. The package order is important, as the packages will +# be installed in the given order. +#export BASE_INCLUDE="$BASEDIR"/data/$CODENAME/base_include + +# File with list of packages to exclude as above. +#export BASE_EXCLUDE="$BASEDIR"/data/$CODENAME/base_exclude + +# Only put the installer onto the cd (set NORECOMMENDS,... as well). +# INSTALLER_CD=0: nothing special (default) +# INSTALLER_CD=1: just add debian-installer (use TASK=debian-installer) +# INSTALLER_CD=2: add d-i and base (use TASK=debian-installer+kernel) +#export INSTALLER_CD=0 + +# Set to 1 to save space by omitting the release notes +# If so we will link to them on the web site. +export OMIT_RELEASE_NOTES=1 + +# Set this to override the defaul location +#export RELEASE_NOTES_LOCATION="http://www.debian.org/releases/$CODENAME" + +case "$OFFICIAL"x in + "Official"x) + export OFFICIAL_VAL=2 + ;; + "Official Beta"x|"Official Snapshot"x) + export OFFICIAL_VAL=1 + ;; + *) + export OFFICIAL_VAL=0 + ;; +esac diff --git a/contrib/common.sh b/contrib/common.sh new file mode 100644 index 00000000..a117aa89 --- /dev/null +++ b/contrib/common.sh @@ -0,0 +1,69 @@ +# Common handy shell script functions + +now () { + date -u +%F:%H:%M:%S +} + +calc_time () { + echo $1 $2 | awk ' + { + split($1, start, ":") + start_time = (3600*start[2]) + (60*start[3]) + start[4] + split($2, end, ":") + end_time = (3600*end[2]) + (60*end[3]) + end[4] + # Cope with going to a new day; do not worry about more than 1 day! + if (start[1] != end[1]) { end_time += 86400 } + time_taken = end_time - start_time + hours = int(time_taken / 3600) + time_taken -= (hours * 3600) + minutes = int(time_taken / 60) + time_taken -= (minutes * 60) + seconds = time_taken + printf("%dh%2.2dm%2.2ds\n", hours, minutes, seconds) + }' +} + +build_started () { + export BUILDNAME=$1 + BUILDS_RUNNING="$BUILDS_RUNNING $BUILDNAME" + export ${BUILDNAME}START=`now` +} + +build_finished () { + ARCH="$1" + BUILDNAME="$2" + BUILDNAMESTART="${BUILDNAME}START" + start=${!BUILDNAMESTART} + + . $PUBDIRJIG/$ARCH/$BUILDNAME-trace + + time_spent=`calc_time $start $end` + echo " $ARCH $BUILDNAME build started at $start, ended at $end (took $time_spent), error $error" + if [ $error -ne 0 ] ; then + arch_error="$arch_error "$BUILDNAME"FAIL/$error/$end" + fi +} + +catch_parallel_builds () { + # Catch parallel builds here + while [ "$BUILDS_RUNNING"x != ""x ] ; do + BUILDS_STILL_RUNNING="" + for BUILDNAME in $BUILDS_RUNNING; do + if [ -e $PUBDIRJIG/$arch/$BUILDNAME-trace ] ; then + build_finished $arch $BUILDNAME + else + BUILDS_STILL_RUNNING="$BUILDS_STILL_RUNNING $BUILDNAME" + fi + done + BUILDS_RUNNING=$BUILDS_STILL_RUNNING + if [ "$BUILDS_RUNNING"x != ""x ] ; then + sleep 1 + fi + done + if [ "$arch_error"x = ""x ] ; then + arch_error="none" + fi + arch_end=`now` + arch_time=`calc_time $arch_start $arch_end` + echo "$ARCH build started at $arch_start, ended at $arch_end (took $arch_time), error(s) $arch_error" +} diff --git a/contrib/generate_firmware_images b/contrib/generate_firmware_images new file mode 100755 index 00000000..7e700a7b --- /dev/null +++ b/contrib/generate_firmware_images @@ -0,0 +1,39 @@ +#!/bin/sh + +set -e + +OUT=/mnt/nfs-cdimage/unofficial/non-free/firmware +TOPDIR=~/build +SUITE=$1 + +. $TOPDIR/CONF.sh + +if [ "$SUITE"x = ""x ] ; then + echo "Need to know what to produce!" + exit 1 +fi + +# Do stuff here! +export TDIR=/org/cdbuilder.debian.org/dst/deb-cd/tmp/firmware +export BASEDIR=$TOPDIR/debian-cd +mkdir -p $TDIR +fakeroot $BASEDIR/tools/make-firmware-image $MIRROR $SUITE $TDIR $BASEDIR + +DATE=`date +%Y%m%d` +mkdir -p $OUT/$SUITE/$DATE +mv $TDIR/firmware*.* $OUT/$SUITE/$DATE/ + +# Update the current link, and delete the old surplus builds. Keep up +# to 3 +cd $OUT/$SUITE +rm -f current +ln -sf $DATE current + +echo "Removing old firmware directories:" +NUM=`ls -d 20*/ 2>/dev/null |wc -l` +NUM=$(($NUM-6)) +if [ "$NUM" -gt "0" ] ; then + REMOVE=`ls -1d 20* 2>/dev/null|head -n $NUM` + echo " $REMOVE" + rm -rf $REMOVE +fi diff --git a/contrib/iso_run b/contrib/iso_run new file mode 100755 index 00000000..3c7df9c8 --- /dev/null +++ b/contrib/iso_run @@ -0,0 +1,93 @@ +#!/bin/sh +# +# Helper for image creation +# +PUBDIRJIG=$1 +RSYNC_TARGET=$2 +ARCH=$3 + +LOG=~/build/log/$ARCH.iso_run +HOSTNAME=`hostname -f` +LOCK=~/iso_run.lock +START=`date -u +%H:%M:%S` + +# Check to see if another sync is in progress +if lockfile -! -l 43200 -r-1 "$LOCK"; then + echo $HOSTNAME is not doing another iso_run, lock file $LOCK exists + exit 1 +fi +trap "rm -f $LOCK > /dev/null 2>&1" exit + +rm -f $LOG + +# Given an ISO image: +# 1. create the torrent file +# 2. create the list file +# 3. copy all of them into place +process_iso() { + FILE=$1 + OUTDIR=$2 + + BTFILE=`echo $FILE.torrent | sed 's/iso-/bt-/'` + ~/build/mktorrent $FILE >> $LOG + + LISTFILE=`echo $FILE | sed 's/iso-/list-/g;s/\.iso$/.list.gz/g'` + ~/build/mklist $FILE >> $LOG + + echo $OUTDIR/$FILE >> $LOG + cp -a $FILE $OUTDIR/$FILE + echo $OUTDIR/$BTFILE >> $LOG + cp -a $BTFILE $OUTDIR/$BTFILE + echo $OUTDIR/$LISTFILE >> $LOG + cp -a $LISTFILE $OUTDIR/$LISTFILE +} + +# Poor man's rsync, but with some local optimisations +copy_files() { + SRC=$1 + TARGET=$2 + ARCHES=$3 + + CURRENT=`pwd` + cd $SRC + for ARCH in $ARCHES + do + for DISKTYPE in dvd cd bd; do + for DIRTYPE in bt iso jigdo list; do + mkdir -p -m775 $TARGET/$ARCH.tmp/$DIRTYPE-$DISKTYPE + done + done + + cd $ARCH + find . -name '*.jigdo' -o -name '*.template' -o -name '*SUMS*' | \ + xargs tar cf - | (cd $TARGET/$ARCH.tmp/ && tar xvf -) >> $LOG + + rm -rf bt-* list-* + mkdir bt-cd bt-dvd bt-bd list-cd list-dvd list-bd + for file in iso-*/*.iso; do + if [ -e $file ] ; then + process_iso $file $TARGET/$ARCH.tmp + fi + done + DATE=`date -u` + sed "s/ARCH/$ARCH/g;s/DATE/$DATE/g" ~/build/weekly.html \ + > $TARGET/$ARCH.tmp/HEADER.html + cd .. + done + cd $CURRENT +} + +copy_files $PUBDIRJIG $RSYNC_TARGET $ARCH + +echo "$START: Starting $ARCH sync from $PUBDIRJIG to $RSYNC_TARGET" >> $LOG +mv $RSYNC_TARGET/$ARCH $RSYNC_TARGET/$ARCH.old +mv $RSYNC_TARGET/$ARCH.tmp $RSYNC_TARGET/$ARCH +rm -rf $RSYNC_TARGET/$ARCH.old & + +# Update the trace file now to force a sync on free.hands.com after each arch +mkdir -p $RSYNC_TARGET/trace +date -u > $RSYNC_TARGET/trace/cdimage.debian.org + +END=`date -u +%H:%M:%S` +echo "$ARCH synced across; started at $START, ended at $END" +echo "$END: Finished" >> $LOG diff --git a/contrib/merge-sums b/contrib/merge-sums new file mode 100755 index 00000000..58c36871 --- /dev/null +++ b/contrib/merge-sums @@ -0,0 +1,40 @@ +#!/bin/sh + +export LANG=C +export LC_ALL=C + +if [ "$1"x = ""x ] ; then + echo "$0: tell me which dir to target!" + echo "abort" + exit 1 +fi + +CHECK=`ls -al $1/*/iso-* 2>/dev/null` +if [ "$CHECK"x = ""x ] ; then + echo "$0: could not find stuff to work on in $1" + echo "abort" + exit 1 +fi + +for DIR in $1/*/*-*; do + cd $DIR + CHECK=`ls -al *SUMS.* 2>/dev/null` + if [ "$CHECK"x != ""x ] ; then + cat MD5SUMS.* | sort -k2 | uniq > MD5SUMS + cat SHA1SUMS.* | sort -k2 | uniq > SHA1SUMS + cat SHA256SUMS.* | sort -k2 | uniq > SHA256SUMS + cat SHA512SUMS.* | sort -k2 | uniq > SHA512SUMS + for ISOFILE in `cat MD5SUMS SHA1SUMS | awk '{print $2}'`; do + JIGDOFILE=${ISOFILE%iso}jigdo + TEMPLATEFILE=${ISOFILE%iso}template + if [ ! -e $ISOFILE ] && [ ! -e $JIGDOFILE ] && [ ! -e $TEMPLATEFILE ] ; then + echo "$ISOFILE/$JIGDOFILE/$TEMPLATEFILE missing in $DIR!" +# echo "abort" +# exit 1 + fi + done + echo "$DIR done" + else + echo "Ignoring $DIR: no checksum files" + fi +done diff --git a/contrib/mklist b/contrib/mklist new file mode 100755 index 00000000..fb42a7eb --- /dev/null +++ b/contrib/mklist @@ -0,0 +1,17 @@ +#!/bin/sh + +for FILE in $@; do + if [ -e "$FILE" ] ; then + ISODIR=`dirname $FILE` + LISTDIR=`echo $ISODIR | sed 's/iso-/list-/'` + if [ ! -d $LISTDIR ] ; then + mkdir -p $LISTDIR + fi + LISTFILE=`echo $FILE | sed 's/\.iso$/.list.gz/g'` + + isoinfo -fR -i $FILE | grep /pool/ | \ + awk '/\.deb$|\.udeb$|\.tar\.gz$|\.dsc$|\.diff\.gz$/ {gsub(".*/","");print $1}' | \ + LANG=C sort | gzip -9 > $LISTFILE + mv $LISTFILE $LISTDIR + fi +done diff --git a/contrib/mktorrent b/contrib/mktorrent new file mode 100755 index 00000000..674d0e1c --- /dev/null +++ b/contrib/mktorrent @@ -0,0 +1,14 @@ +#!/bin/sh + +for FILE in $@; do + MKTORRENT=/home/debian-cd/bt/btmakemetafile.py + ISODIR=`dirname $FILE` + BTDIR=`echo $ISODIR | sed 's/iso-/bt-/;s/usb-/bt-/'` + if [ ! -d $BTDIR ] ; then + mkdir -p $BTDIR + fi + $MKTORRENT http://bttracker.debian.org:6969/announce \ + --comment '"Debian CD from cdimage.debian.org"' \ + $FILE | grep -v complete + mv $FILE.torrent $BTDIR +done diff --git a/contrib/report_build_error b/contrib/report_build_error new file mode 100755 index 00000000..f51a15e0 --- /dev/null +++ b/contrib/report_build_error @@ -0,0 +1,38 @@ +#!/bin/sh + +OUTDIR=$1 +ARCH=$2 +RESULTS=$3 + +OUTFILE=$OUTDIR/$ARCH/HEADER.html + +grep -q WARNING $OUTFILE +if [ $? -ne 0 ] ; then + echo >> $OUTFILE + echo "

WARNING:" >> $OUTFILE + echo "This build is not up to date; it is the most recent successful build.

" >> $OUTFILE + echo >> $OUTFILE + echo "

Later weekly builds failed with errors:

" >> $OUTFILE +fi + +echo "
" >> $OUTFILE +echo "" >> $OUTFILE +