From 68177cbb8668478d63c92619b2119c200c3a7517 Mon Sep 17 00:00:00 2001 From: Steve McIntyre <93sam@debian.org> Date: Sun, 14 Nov 2010 15:17:33 +0000 Subject: [PATCH] current set of scripts in use on pettersson --- contrib/cronjob.daily | 225 +++++++++++++++++++++++++++-------------- contrib/cronjob.weekly | 159 +++++------------------------ contrib/ftp.cron | 110 ++++++++++++++++++++ contrib/testingcds | 83 ++++++++++----- 4 files changed, 343 insertions(+), 234 deletions(-) create mode 100755 contrib/ftp.cron diff --git a/contrib/cronjob.daily b/contrib/cronjob.daily index fd5946bf..0a4fc8a4 100755 --- a/contrib/cronjob.daily +++ b/contrib/cronjob.daily @@ -5,8 +5,11 @@ TOPDIR=$(dirname $0) . $TOPDIR/settings.sh export PUBDIRJIG=$PUBDIR/daily-builds -export PUBDIRISO=$PUBDIR/daily-builds export DATE_BUILD="$DATE-$BUILDNUM" +export TESTING_SUITE=squeeze +BUILDS_RUNNING="" + +. $TOPDIR/common.sh # If we're doing a normal set of daily/weekly builds, leave the # checksum filenames alone. Otherwise, make life easier for people @@ -18,31 +21,52 @@ fi finalise_arch_dir () { ARCH=$1 INST_VER=$2 + TYPE=$3 - ARCH_ISO_DIR=$PUBDIRISO/${INST_VER}_d-i/$DATE_BUILD/$ARCH/iso-cd - ARCH_JIGDO_DIR=$PUBDIRISO/${INST_VER}_d-i/$DATE_BUILD/$ARCH/jigdo-cd - if [ -e $ARCH_ISO_DIR ] ; then - cd $ARCH_ISO_DIR - for file in *businesscard*iso *netinst*iso; do - zsyncmake -e -u $file $file > /dev/null - done - DATESTRING=`date -u` - sed "s/ARCH/$ARCH/g;s/DATE/$DATESTRING/g;s/BUILDNUM/$BUILDNUM/g;s/INST_VER/$INST_VER/g" $TOPDIR/daily.html > HEADER.html - $TOPDIR/debian-cd/tools/imagesums $ARCH_JIGDO_DIR $SUMS_EXTENSION - cp $ARCH_JIGDO_DIR/*SUMS* $ARCH_ISO_DIR - cd .. - ~/build/mktorrent iso-cd/*iso - ~/build/mklist iso-cd/*iso - cd $TOPDIR + if [ "$TYPE" = "free" ] ; then + ARCH_DIR=$PUBDIRJIG/${INST_VER}_d-i/$DATE_BUILD/$ARCH/ + elif [ "$TYPE" = "firmware" ] ; then + ARCH_DIR=$PUBDIRJIG-firmware/${INST_VER}_d-i/$DATE_BUILD/$ARCH else - echo "finalise_arch_dir: $ARCH_ISO_DIR does not exist!" + echo "Uncaught finalise_arch_dir TYPE \"$TYPE\"" + exit 1 + fi + + # Only keep things / do any work here if there are any files in the ARCH_DIR + NUM_FILES=`find $ARCH_DIR -type f | wc -l` + if [ $NUM_FILES = 0 ] ; then + rm -rf $ARCH_DIR + else + ARCH_ISO_DIR=${ARCH_DIR}/iso-cd + ARCH_JIGDO_DIR=${ARCH_DIR}/jigdo-cd + + cd $ARCH_ISO_DIR + DATESTRING=`date -u` + if [ "$TYPE" = "free" ] ; then + sed "s/ARCH/$ARCH/g;s/DATE/$DATESTRING/g;s/BUILDNUM/$BUILDNUM/g;s/INST_VER/$INST_VER/g" $TOPDIR/daily.html > HEADER.html + else + sed "s/ARCH/$ARCH/g;s/DATE/$DATESTRING/g;s/BUILDNUM/$BUILDNUM/g;s/INST_VER/$INST_VER/g" $TOPDIR/daily-firmware.html > HEADER.html + fi + $TOPDIR/debian-cd/tools/imagesums $ARCH_JIGDO_DIR $SUMS_EXTENSION + cp $ARCH_JIGDO_DIR/*SUMS* $ARCH_ISO_DIR + cd .. + ~/build/mktorrent iso-cd/*iso + ~/build/mklist iso-cd/*iso + cd $TOPDIR fi } -export RSYNC_TARGET=/mnt/nfs-cdimage/daily-builds +export RSYNC_TARGET_FREE=/mnt/nfs-cdimage/daily-builds +export RSYNC_TARGET_FIRMWARE=/mnt/nfs-cdimage/unofficial/non-free/cd-including-firmware if [ "$ARCHES"x = ""x ] ; then - ARCHES="amd64 armel hppa i386 ia64 mips mipsel powerpc sparc multi-arch" + ARCHES="amd64 armel hppa i386 ia64 mips mipsel powerpc sparc multi-arch kfreebsd-amd64 kfreebsd-i386" +# ARCHES="amd64" +fi + +if [ "$ARCHES_FIRMWARE"x = ""x ] ; then + ARCHES_FIRMWARE="amd64 i386 powerpc multi-arch" +# ARCHES_FIRMWARE="amd64" fi cd $TOPDIR && @@ -51,6 +75,8 @@ if lockfile -r0 .debian-cd.lock ; then echo "svn update debian-cd:" cd debian-cd && svn cleanup ; svn up ; cd .. + cd $TOPDIR + . images4testing_d-i # Allow desktop selection in isolinux menu for i386 and amd64 @@ -59,95 +85,134 @@ if lockfile -r0 .debian-cd.lock ; then export KERNEL_PARAMS="desktop=all" for arch in $ARCHES; do + rm -rf $PUBDIRJIG/$arch + arch_error="" + arch_start=`now` + echo "Building $arch:" + for ARCHDIR in \ + ${PUBDIRJIG}/sid_d-i/${DATE_BUILD}/${arch} \ + ${PUBDIRJIG}/${TESTING_SUITE}_d-i/${DATE_BUILD}/${arch} \ + ${PUBDIRJIG}-firmware/sid_d-i/${DATE_BUILD}/${arch} \ + ${PUBDIRJIG}-firmware/${TESTING_SUITE}_d-i/${DATE_BUILD}/${arch} \ + ; do + mkdir -p ${ARCHDIR}/iso-cd ${ARCHDIR}/jigdo-cd + done + if [ "$arch" = "multi-arch" ] ; then + for arch1 in $ARCHES_FIRMWARE; do + if [ "$arch" = "$arch1" ] ; then + build_started SIDNIFIRMWARE + OMIT_RELEASE_NOTES=1 OMIT_MANUAL=1 NORECOMMENDS=1 \ + NOSUGGESTS=1 COMPLETE=0 INSTALLER_CD=C \ + FORCE_FIRMWARE=1 CDNAME=firmware \ + TASK=debian-installer+kernel LOGAPPEND="-1" \ + MAXISOS=ALL MAXJIGDOS=ALL \ + DI=sid DI_DIST="$DI_DIST" VARIANTS=xen \ + DI_WWW_HOME=default ./testingcds "amd64 i386 powerpc" & - echo " i386/amd64/ppc sid netinst WITH FIRMWARE" - OMIT_RELEASE_NOTES=1 OMIT_MANUAL=1 NORECOMMENDS=1 \ - NOSUGGESTS=1 COMPLETE=0 INSTALLER_CD=C \ - FORCE_FIRMWARE=1 CDNAME=firmware \ - TASK=debian-installer+kernel LOGAPPEND="-1" \ - MAXISOS=ALL MAXJIGDOS=ALL \ - DI=sid DI_DIST="$DI_DIST" VARIANTS=xen \ - DI_WWW_HOME=default ./testingcds "amd64 i386 powerpc" - rm -rf $PUBDIRISO/sid_d-i/$DATE_BUILD/multi-arch - mv -v $PUBDIRISO/sid_d-i/$DATE_BUILD/multi $PUBDIRISO/sid_d-i/$DATE_BUILD/multi-arch + build_started TESTINGNIFIRMWARE + OMIT_RELEASE_NOTES=1 OMIT_MANUAL=1 NORECOMMENDS=1 \ + NOSUGGESTS=1 COMPLETE=0 INSTALLER_CD=C \ + FORCE_FIRMWARE=1 CDNAME=firmware \ + TASK=debian-installer+kernel LOGAPPEND="-1" \ + MAXISOS=ALL MAXJIGDOS=ALL \ + DI=${TESTING_SUITE} DI_DIST="$DI_DIST" VARIANTS=xen \ + ./testingcds "amd64 i386 powerpc" & + fi + done - echo " i386/amd64/ppc sid netinst" + build_started SIDNI OMIT_RELEASE_NOTES=1 OMIT_MANUAL=1 NORECOMMENDS=1 \ NOSUGGESTS=1 COMPLETE=0 INSTALLER_CD=2 \ TASK=debian-installer+kernel LOGAPPEND="-1" \ MAXISOS=ALL MAXJIGDOS=ALL \ DI=sid DI_DIST="$DI_DIST" VARIANTS=xen \ - DI_WWW_HOME=default ./testingcds "amd64 i386 powerpc" - cp -av $PUBDIRISO/sid_d-i/$DATE_BUILD/multi/. $PUBDIRISO/sid_d-i/$DATE_BUILD/multi-arch - rm -rf $PUBDIRISO/sid_d-i/$DATE_BUILD/multi - finalise_arch_dir $arch sid + DI_WWW_HOME=default ./testingcds "amd64 i386 powerpc" & - echo " i386/amd64/ppc squeeze netinst" + build_started TESTINGNI OMIT_RELEASE_NOTES=1 OMIT_MANUAL=1 NORECOMMENDS=1 \ NOSUGGESTS=1 COMPLETE=0 INSTALLER_CD=2 \ TASK=debian-installer+kernel LOGAPPEND="-1" \ MAXISOS=ALL MAXJIGDOS=ALL \ - DI=squeeze DI_DIST="$DI_DIST" VARIANTS=xen \ - ./testingcds "amd64 i386 powerpc" - rm -rf $PUBDIRISO/squeeze_d-i/$DATE_BUILD/multi-arch - mv -v $PUBDIRISO/squeeze_d-i/$DATE_BUILD/multi $PUBDIRISO/squeeze_d-i/$DATE_BUILD/multi-arch - finalise_arch_dir $arch squeeze + DI=${TESTING_SUITE} DI_DIST="$DI_DIST" VARIANTS=xen \ + ./testingcds "amd64 i386 powerpc" & + + catch_parallel_builds + rm -rf $PUBDIRJIG/$arch + if [ "$arch_error"x = "none"x ] ; then + finalise_arch_dir $arch sid free + finalise_arch_dir $arch ${TESTING_SUITE} free + finalise_arch_dir $arch sid firmware + finalise_arch_dir $arch ${TESTING_SUITE} firmware + fi else # end of m-a - case $arch in - i386|amd64|powerpc) - echo " $arch sid netinst WITH FIRMWARE" + for arch1 in $ARCHES_FIRMWARE; do + if [ "$arch" = "$arch1" ] ; then + build_started SIDNIFIRMWARE OMIT_RELEASE_NOTES=1 OMIT_MANUAL=1 NORECOMMENDS=1 \ FORCE_FIRMWARE=1 CDNAME=firmware \ NOSUGGESTS=1 COMPLETE=0 INSTALLER_CD=C \ TASK=debian-installer+kernel MAXISOS=ALL MAXJIGDOS=ALL \ DI=sid DI_WWW_HOME=default DI_DIR="$ARCH_DI_DIR" \ - ./testingcds "$arch" - ;; - esac - - echo " $arch sid bc" - OMIT_RELEASE_NOTES=1 OMIT_MANUAL=1 NORECOMMENDS=1 \ - NOSUGGESTS=1 COMPLETE=0 INSTALLER_CD=1 \ - TASK=debian-installer MAXISOS=ALL MAXJIGDOS=ALL \ - DI=sid DI_WWW_HOME=default DI_DIR="$ARCH_DI_DIR" \ - ./testingcds "$arch" - echo " $arch sid netinst" - OMIT_RELEASE_NOTES=1 OMIT_MANUAL=1 NORECOMMENDS=1 \ - NOSUGGESTS=1 COMPLETE=0 INSTALLER_CD=2 \ - TASK=debian-installer+kernel MAXISOS=ALL MAXJIGDOS=ALL \ - DI=sid DI_WWW_HOME=default DI_DIR="$ARCH_DI_DIR" \ - ./testingcds "$arch" - finalise_arch_dir $arch sid + ./testingcds "$arch" & - echo " $arch squeeze bc" - OMIT_RELEASE_NOTES=1 OMIT_MANUAL=1 NORECOMMENDS=1 \ + build_started TESTINGNIFIRMWARE + OMIT_RELEASE_NOTES=1 OMIT_MANUAL=1 NORECOMMENDS=1 \ + FORCE_FIRMWARE=1 CDNAME=firmware \ + NOSUGGESTS=1 COMPLETE=0 INSTALLER_CD=C \ + TASK=debian-installer+kernel MAXISOS=ALL MAXJIGDOS=ALL \ + DI=${TESTING_SUITE} DI_DIST="$DI_DIST" \ + ./testingcds "$arch" & + fi + done + + build_started SIDBC + OMIT_RELEASE_NOTES=1 OMIT_MANUAL=1 NORECOMMENDS=1 \ NOSUGGESTS=1 COMPLETE=0 INSTALLER_CD=1 \ TASK=debian-installer MAXISOS=ALL MAXJIGDOS=ALL \ - DI=squeeze DI_DIST="$DI_DIST" \ - ./testingcds "$arch" - echo " $arch squeeze netinst" - OMIT_RELEASE_NOTES=1 OMIT_MANUAL=1 NORECOMMENDS=1 \ + DI=sid DI_WWW_HOME=default DI_DIR="$ARCH_DI_DIR" \ + ./testingcds "$arch" & + + build_started SIDNI + OMIT_RELEASE_NOTES=1 OMIT_MANUAL=1 NORECOMMENDS=1 \ NOSUGGESTS=1 COMPLETE=0 INSTALLER_CD=2 \ TASK=debian-installer+kernel MAXISOS=ALL MAXJIGDOS=ALL \ - DI=squeeze DI_DIST="$DI_DIST" \ - ./testingcds "$arch" - finalise_arch_dir $arch squeeze + DI=sid DI_WWW_HOME=default DI_DIR="$ARCH_DI_DIR" \ + ./testingcds "$arch" & + + build_started TESTINGBC + OMIT_RELEASE_NOTES=1 OMIT_MANUAL=1 NORECOMMENDS=1 \ + NOSUGGESTS=1 COMPLETE=0 INSTALLER_CD=1 \ + TASK=debian-installer MAXISOS=ALL MAXJIGDOS=ALL \ + DI=${TESTING_SUITE} DI_DIST="$DI_DIST" \ + ./testingcds "$arch" & + + build_started TESTINGNI + OMIT_RELEASE_NOTES=1 OMIT_MANUAL=1 NORECOMMENDS=1 \ + NOSUGGESTS=1 COMPLETE=0 INSTALLER_CD=2 \ + TASK=debian-installer+kernel MAXISOS=ALL MAXJIGDOS=ALL \ + DI=${TESTING_SUITE} DI_DIST="$DI_DIST" \ + ./testingcds "$arch" & + + catch_parallel_builds + rm -rf $PUBDIRJIG/$arch + if [ "$arch_error"x = "none"x ] ; then + finalise_arch_dir $arch sid free + finalise_arch_dir $arch ${TESTING_SUITE} free + finalise_arch_dir $arch sid firmware + finalise_arch_dir $arch ${TESTING_SUITE} firmware + fi fi # end of normal arch build done if [ "$NOSYNC"x = ""x ] ; then - echo "Sync across to output dir: $RSYNC_TARGET" - rsync -rHltv $PUBDIRISO/ $RSYNC_TARGET/ - echo "Remove extra dailies:" - ~/bin/remove_extra_dailies echo "Clean up old builds on build machine:" - for DIR in $PUBDIRISO/* ; do + for DIR in $PUBDIRJIG/* $PUBDIRJIG-firmware/* ; do cd $DIR for THIS in 20* ; do if [ $THIS != $DATE-$BUILDNUM ] ; then @@ -157,7 +222,17 @@ if lockfile -r0 .debian-cd.lock ; then done done - echo "$DATE-$BUILDNUM" > $RSYNC_TARGET/daily-trace + echo "Sync normal builds across to output dir: $RSYNC_TARGET_FREE" + rsync -rHltv $PUBDIRJIG/ $RSYNC_TARGET_FREE/ + echo "Remove extra builds for normal dailies:" + ~/bin/remove_extra_dailies $RSYNC_TARGET_FREE "$ARCHES" + + echo "Sync firmware builds across to output dir: $RSYNC_TARGET_FIRMWARE" + rsync -rHltv $PUBDIRJIG-firmware/ $RSYNC_TARGET_FIRMWARE/ + echo "Remove extra builds for firmware dailies:" + ~/bin/remove_extra_dailies $RSYNC_TARGET_FIRMWARE "$ARCHES_FIRMWARE" + + echo "$DATE-$BUILDNUM" > $RSYNC_TARGET_FREE/daily-trace fi cd $TOPDIR diff --git a/contrib/cronjob.weekly b/contrib/cronjob.weekly index de68dd47..201c7740 100755 --- a/contrib/cronjob.weekly +++ b/contrib/cronjob.weekly @@ -5,14 +5,11 @@ export TOPDIR=$(dirname $0) . $TOPDIR/settings.sh export PUBDIRJIG=$PUBDIR/weekly-builds -export PUBDIRISO=$PUBDIR/weekly-builds export DATE_BUILD="$DATE-$BUILDNUM" -export RSYNC_TARGET=/mnt/nfs-cdimage/weekly-builds +. $TOPDIR/common.sh -if [ "$PARALLEL"x = ""x ] ; then - PARALLEL=0 -fi +export RSYNC_TARGET=/mnt/nfs-cdimage/weekly-builds # If we're doing a normal set of daily/weekly builds, leave the # checksum filenames alone. Otherwise, make life easier for people @@ -40,46 +37,8 @@ export OMIT_RELEASE_NOTES BUILDS_RUNNING="" -now () { - date -u +%F:%H:%M:%S -} - -calc_time () { - echo $1 $2 | awk ' - { - split($1, start, ":") - start_time = (3600*start[2]) + (60*start[3]) + start[4] - split($2, end, ":") - end_time = (3600*end[2]) + (60*end[3]) + end[4] - # Cope with going to a new day; do not worry about more than 1 day! - if (start[1] != end[1]) { end_time += 86400 } - time_taken = end_time - start_time - hours = int(time_taken / 3600) - time_taken -= (hours * 3600) - minutes = int(time_taken / 60) - time_taken -= (minutes * 60) - seconds = time_taken - printf("%dh%2.2dm%2.2ds\n", hours, minutes, seconds) - }' -} - -build_finished () { - ARCH="$1" - BUILDNAME="$2" - BUILDNAMESTART="${BUILDNAME}START" - start=${!BUILDNAMESTART} - - . $PUBDIRJIG/$ARCH/$BUILDNAME-trace - - time_spent=`calc_time $start $end` - echo "$ARCH $BUILDNAME build started at $start, ended at $end (took $time_spent), error $error" - if [ $error -ne 0 ] ; then - arch_error="$arch_error "$BUILDNAME"FAIL/$error/$end" - fi -} - if [ "$ARCHES"x = ""x ] ; then - ARCHES="i386 source amd64 multi powerpc armel hppa ia64 mips mipsel s390 sparc" + ARCHES="i386 source amd64 multi powerpc armel hppa ia64 mips mipsel s390 sparc kfreebsd-amd64 kfreebsd-i386" fi if lockfile -r0 $TOPDIR/.debian-cd.lock ; then @@ -114,16 +73,9 @@ if lockfile -r0 $TOPDIR/.debian-cd.lock ; then *) export MAXISOS=3; export MAXJIGDOS=ALL ;; esac - CDSTART=`now` unset DESKTOP - export BUILDNAME="CD" - BUILDS_RUNNING="$BUILDS_RUNNING $BUILDNAME" - if [ $PARALLEL = 1 ] ; then - ./testingcds "$arch" & - else - ./testingcds "$arch" - build_finished $arch $BUILDNAME - fi + build_started CD + ./testingcds "$arch" & fi # Full DVD set @@ -133,22 +85,13 @@ if lockfile -r0 $TOPDIR/.debian-cd.lock ; then i386|amd64|source) export MAXISOS=ALL; export MAXJIGDOS=ALL ;; *) - export MAXISOS=ALL; export MAXJIGDOS=ALL ;; + export MAXISOS=1; export MAXJIGDOS=ALL ;; esac - DVDSTART=`now` export DESKTOP=all - export BUILDNAME="DVD" - BUILDS_RUNNING="$BUILDS_RUNNING $BUILDNAME" - if [ $PARALLEL = 1 ] ; then - INSTALLER_CD=3 TASK=Debian-all \ - KERNEL_PARAMS='desktop=all' \ - ./testingcds "$arch" & - else - INSTALLER_CD=3 TASK=Debian-all \ - KERNEL_PARAMS='desktop=all' \ - ./testingcds "$arch" - build_finished $arch $BUILDNAME - fi + build_started DVD + INSTALLER_CD=3 TASK=Debian-all \ + KERNEL_PARAMS='desktop=all' \ + ./testingcds "$arch" & fi # Full BD set @@ -158,94 +101,44 @@ if lockfile -r0 $TOPDIR/.debian-cd.lock ; then i386|amd64|source) export MAXISOS=0 export MAXJIGDOS=ALL - BDSTART=`now` export DESKTOP=all - export BUILDNAME="BD" - BUILDS_RUNNING="$BUILDS_RUNNING $BUILDNAME" - if [ $PARALLEL = 1 ] ; then - INSTALLER_CD=9 TASK=Debian-all \ - KERNEL_PARAMS='desktop=all' \ - ./testingcds "$arch" & - else - INSTALLER_CD=9 TASK=Debian-all \ - KERNEL_PARAMS='desktop=all' \ - ./testingcds "$arch" - build_finished $arch $BUILDNAME - fi + build_started BD + INSTALLER_CD=9 TASK=Debian-all \ + KERNEL_PARAMS='desktop=all' \ + ./testingcds "$arch" & ;; *) - echo "Not running BD build for $arch" + echo " Not running BD build for $arch" ;; esac fi # kde and xfce/lxde cds if [ "$arch"x != "source"x ] && [ "$NOCD"x = ""x ] ; then - KDECDSTART=`now` export MAX_PKG_SIZE=300000000 export DESKTOP=kde export MAXISOS=1; export MAXJIGDOS=1 - export BUILDNAME="KDECD" - BUILDS_RUNNING="$BUILDS_RUNNING $BUILDNAME" - if [ $PARALLEL = 1 ] ; then + build_started KDECD INSTALLER_CD=4 TASK=Debian-kde \ KERNEL_PARAMS='desktop=kde' \ MAXCDS=1 MAXISOS=ALL MAXJIGDOS=ALL \ ./testingcds "$arch" & - else - INSTALLER_CD=4 TASK=Debian-kde \ - KERNEL_PARAMS='desktop=kde' \ - MAXCDS=1 MAXISOS=ALL MAXJIGDOS=ALL \ - ./testingcds "$arch" - build_finished $arch $BUILDNAME - fi - LIGHTCDSTART=`now` export MAX_PKG_SIZE=300000000 export DESKTOP=light export MAXISOS=1; export MAXJIGDOS=1 - export BUILDNAME="LIGHTCD" - BUILDS_RUNNING="$BUILDS_RUNNING $BUILDNAME" - if [ $PARALLEL = 1 ] ; then - INSTALLER_CD=B TASK=Debian-light \ - KERNEL_PARAMS='desktop=light' \ - MAXCDS=1 MAXISOS=ALL MAXJIGDOS=ALL \ - ./testingcds "$arch" & - else - INSTALLER_CD=B TASK=Debian-light \ - KERNEL_PARAMS='desktop=light' \ - MAXCDS=1 MAXISOS=ALL MAXJIGDOS=ALL \ - ./testingcds "$arch" - build_finished $arch $BUILDNAME - fi + build_started LIGHTCD + INSTALLER_CD=B TASK=Debian-light \ + KERNEL_PARAMS='desktop=light' \ + MAXCDS=1 MAXISOS=ALL MAXJIGDOS=ALL \ + ./testingcds "$arch" & fi - # Catch parallel builds here - while [ "$BUILDS_RUNNING"x != ""x ] ; do - BUILDS_STILL_RUNNING="" - for BUILDNAME in $BUILDS_RUNNING; do - if [ -e $PUBDIRJIG/$arch/$BUILDNAME-trace ] ; then - build_finished $arch $BUILDNAME - else - BUILDS_STILL_RUNNING="$BUILDS_STILL_RUNNING $BUILDNAME" - fi - done - BUILDS_RUNNING=$BUILDS_STILL_RUNNING - if [ "$BUILDS_RUNNING"x != ""x ] ; then - sleep 1 - fi - done - arch_end=`now` - - if [ "$arch_error"x = ""x ] ; then - arch_error="none" - fi - arch_time=`calc_time $arch_start $arch_end` - echo "$ARCH build started at $arch_start, ended at $arch_end (took $arch_time), error(s) $arch_error" + catch_parallel_builds if [ "$NOSYNC"x = ""x ] ; then if [ "$arch_error"x = "none"x ] ; then - echo "Running ~/build/iso_run $PUBDIRJIG/ $RSYNC_TARGET/ $arch &" + echo " Running ~/build/iso_run $PUBDIRJIG/ $RSYNC_TARGET/ $arch &" ~/build/iso_run $PUBDIRJIG/ $RSYNC_TARGET/ $arch & else ~/build/report_build_error $RSYNC_TARGET $arch "$arch_error" @@ -259,12 +152,12 @@ if lockfile -r0 $TOPDIR/.debian-cd.lock ; then if [ "$NODVD"x = ""x ] ; then export MAX_PKG_SIZE=999999999999 - DVDSTART=`now` export DESKTOP=all - export BUILDNAME="DVD" + build_started DVD INSTALLER_CD=6 TASK=Debian-all \ KERNEL_PARAMS='desktop=all' \ MAXCDS=1 MAXISOS=ALL MAXJIGDOS=ALL \ + VARIANTS=xen \ ./testingcds "i386 amd64 source" # We don't do multi in parallel, only one build type # to do! @@ -287,7 +180,7 @@ if lockfile -r0 $TOPDIR/.debian-cd.lock ; then if [ "$NOSYNC"x = ""x ] ; then if [ "$arch_error"x = "none"x ] || [ "$arch_error"x = ""x ] ; then - echo "Running ~/build/iso_run $PUBDIRJIG/ $RSYNC_TARGET/ multi-arch &" + echo " Running ~/build/iso_run $PUBDIRJIG/ $RSYNC_TARGET/ multi-arch &" ~/build/iso_run $PUBDIRJIG/ $RSYNC_TARGET/ multi-arch & else ~/build/report_build_error $RSYNC_TARGET multi-arch "$arch_error" diff --git a/contrib/ftp.cron b/contrib/ftp.cron new file mode 100755 index 00000000..08c8f5a3 --- /dev/null +++ b/contrib/ftp.cron @@ -0,0 +1,110 @@ +#!/bin/sh + +set -e + +TOPDIR=`pwd` +. $TOPDIR/settings.sh +. $TOPDIR/.ftp.buildnum + +MYCOPY=$TOPDIR/.ftp.cron +BUILDLOCK=$TOPDIR/.debian-cd.lock +MAILLIST=$TOPDIR/.debian-cd.mail + +# Change these to abort daily/weekly builds as appropriate +ABORT_DAILY=n +ABORT_WEEKLY=n + +if [ "$1"x = "-f"x ] ; then + FORCE=1 +fi + +mail_list() { + if [ -e "$MAILLIST" ] ; then + MYEMAIL=`grep -v "^#" "$MAILLIST"` + else + MYEMAIL=`whoami` + fi + for i in $MYEMAIL + do + cat "$MYCOPY"|mail -s "$1" "$i" + done + return 0 +} + +if [ -f "$TRACE" ] ; then + if [ "$FORCE"x = "1"x ] || ! /usr/bin/cmp -s "$MYCOPY".lastbuild "$TRACE" ; then + if [ -e "$BUILDLOCK" ]; then + if ! /usr/bin/cmp -s "$MYCOPY" "$TRACE" ; then + if ! /usr/bin/cmp -s "$MYCOPY" "$MYCOPY".building ; then + mail_list "testingcds missed the daily build for the pulse because of a lock" + fi + cp "$TRACE" "$MYCOPY" + cat "$MYCOPY" >> "$MYCOPY".stats + fi + else + if ! /usr/bin/cmp -s "$MYCOPY" "$TRACE" ; then + cp "$TRACE" "$MYCOPY" + cat "$MYCOPY" >> "$MYCOPY".stats + fi + + # Work out the next build date/number combo + if [ "$LASTDATE"x != "$DATE"x ] ; then + BUILDNUM=1 + else + BUILDNUM=$(($LASTBUILDNUM + 1)) + fi + echo "LASTDATE=$DATE" > $TOPDIR/.ftp.buildnum + echo "LASTBUILDNUM=$BUILDNUM" >> $TOPDIR/.ftp.buildnum + export DATE BUILDNUM + + echo "Last build was $LASTDATE-$LASTBUILDNUM" + echo "New build will be $DATE-$BUILDNUM" + + cp "$MYCOPY" "$MYCOPY".building + + # "Daily" builds + # Only run certain builds, otherwise we're just wasting + # time reproducing identical results: + # - archive changes are likely to be minimal + # - packages only get uploaded at best once daily for most buildds + # - it's fairly unlikely they'll actually affect/improve installs, + # especially for the small images + # - D-I images only get built once or at most twice daily + DAILIES_DESIRED="1 5" + ODD_BUILD=`echo "$BUILDNUM % 2" | bc` + if [ "$ABORT_DAILY"x = "y"x ] && [ "$FORCE"x != "1"x ] ; then + echo "BAILING OUT OF DAILY BUILD" + mail_list "$HOSTNAME BAILING OUT OF DAILY BUILD" + elif [ "$ODD_BUILD" = "0" ] && [ "$FORCE"x != "1"x ] ; then + echo "Not running even-numbered daily build #$BUILDNUM" + else + echo "Running daily build #$BUILDNUM." + if [ "$FORCE"x = "1"x ] ; then + echo "Build forced by hand" + else + echo "Triggered by mirror pulse:" + cat "$TRACE" + fi + $TOPDIR/cronjob.daily + cp "$MYCOPY" "$MYCOPY".lastbuild + if ! /usr/bin/cmp -s "$MYCOPY".lastbuild "$TRACE" ; then + mail_list "testingcds has detected a pulse while we were building dailies" + fi + fi + + # Weekly build + if [ `date +%a` = "Mon" -a "$BUILDNUM"x = "1"x ] ; then + if [ "$ABORT_WEEKLY"x = "y"x ] && [ "$FORCE"x != "1"x ] ; then + echo "BAILING OUT OF WEEKLY BUILD" + mail_list "$HOSTNAME BAILING OUT OF WEEKLY BUILD" + else + $TOPDIR/cronjob.weekly + fi + fi + exit 0 + fi + fi +fi +exit 1 + + diff --git a/contrib/testingcds b/contrib/testingcds index 7115253a..40e8626a 100644 --- a/contrib/testingcds +++ b/contrib/testingcds @@ -12,7 +12,9 @@ #set -x -CONF=~/build/CONF.sh +if [ "$CONF"x = ""x ] ; then + CONF=~/build/CONF.sh +fi export CF=$CONF now () { @@ -40,17 +42,13 @@ fi # $DI is used to set DI_CODENAME and also to specify the directory for images if [ "$DI" ] ; then export DI_CODENAME="$DI" - if [ "$INSTALLER_CD" = "1" -o "$INSTALLER_CD" = "2" -o "$INSTALLER_CD" = "C" ] ; then - PUBDIRJIG="$PUBDIRJIG"/"$DI"_d-i - PUBDIRISO="$PUBDIRISO"/"$DI"_d-i - fi fi export LOG="`pwd`/log/" if [ "$NUM_ARCHES"x = "1"x ] ; then export OUTARCH=$ARCH else - export OUTARCH="multiarch" + export OUTARCH="multi-arch" fi export RUN="$INSTALLER_CD$DI$OUTARCH$LOGAPPEND" export LOGFILE="$LOG/$RUN" @@ -81,21 +79,22 @@ if [ "$OUT_TYPE"x = ""x ] ; then OUT_TYPE=cd fi -if [ "$BUILDNAME"x = ""x ] ; then - TRACEFILE="$OUT_TYPE-trace" -else - TRACEFILE="$BUILDNAME-trace" -fi - export IMAGETARGET=official_images cd debian-cd && . $CONF -echo Making $ARCH "$DISKTYPE" in $OUT if [ "$OUT"x = ""x ] ; then echo Config error exit 1 fi +if [ "$BUILDNAME"x = ""x ] ; then + TRACEFILE="$OUT_TYPE-trace" + echo " Making $ARCH ${DISKTYPE} in $OUT" +else + TRACEFILE="$BUILDNAME-trace" + echo " Making $ARCH ${DISKTYPE} (${BUILDNAME}) in $OUT" +fi + if [ "$INSTALLER_CD" = "1" ] ; then export OFFICIAL="Official BusinessCard Snapshot" elif [ "$INSTALLER_CD" = "2" ] ; then @@ -120,6 +119,7 @@ fi rm -rf "$OUT" mkdir -p "$OUT" +rm -f "$PUBDIRJIG"/$OUTARCH/$TRACEFILE #mkdir -p "$LOG" overview_log="$LOG/overview" @@ -134,34 +134,61 @@ overview () { > $LOGFILE # Do the actual building +START=`now` if ./build.sh "$ARCH" >>"$LOGFILE" 2>&1 ; then # if it went ok get things ready for publishing # we treat the daily netinst images in a different way - if [ "$INSTALLER_CD" = "1" -o "$INSTALLER_CD" = "2" -o "$INSTALLER_CD" = "C" ] ; then + if [ "$INSTALLER_CD" = "1" -o "$INSTALLER_CD" = "2" ] ; then ################################## # NETINST STUFF ################################## - # make the dirs we'll use to publish things - TOPDIR="$PUBDIRISO"/$DATE_BUILD/$OUTARCH - mkdir -p $TOPDIR/iso-cd $TOPDIR/jigdo-cd + ARCHDIR="$PUBDIRJIG"/${DI}_d-i/$DATE_BUILD/$OUTARCH # put the isos and jigdos in their place for i in "$OUT"/*-1.iso; do BASE=`basename $i .iso` NETINSTNAME=`echo $BASE | sed 's/-BC-1/-businesscard/g;s/-NETINST-1/-netinst/g'` - mv -v $OUT/$BASE.iso $TOPDIR/iso-cd/$NETINSTNAME.iso + mv $OUT/$BASE.iso $ARCHDIR/iso-cd/$NETINSTNAME.iso cat $OUT/$BASE.jigdo | \ sed 's/-BC-1/-businesscard/g;s/-NETINST-1/-netinst/g' | \ - gzip --best --no-name > $TOPDIR/jigdo-cd/$NETINSTNAME.jigdo + gzip --best --no-name > $ARCHDIR/jigdo-cd/$NETINSTNAME.jigdo rm $OUT/$BASE.jigdo - mv -v $OUT/$BASE.template $TOPDIR/jigdo-cd/$NETINSTNAME.template + mv $OUT/$BASE.template $ARCHDIR/jigdo-cd/$NETINSTNAME.template # Make sure that the ISO is as new/newer than the jigdo file; #587774 - touch $TOPDIR/iso-cd/$NETINSTNAME.iso + touch $ARCHDIR/iso-cd/$NETINSTNAME.iso done rm -f "$OUT"/* + END=`now` + mkdir -p "$PUBDIRJIG"/$OUTARCH + echo -e "start=$START\nend=$END\nerror=0\n" > "$PUBDIRJIG"/$OUTARCH/$TRACEFILE + elif [ "$INSTALLER_CD" = "C" ] ; then + + ################################## + # FIRMWARE NETINST STUFF + ################################## + + ARCHDIR="$PUBDIRJIG"-firmware/${DI}_d-i/$DATE_BUILD/$OUTARCH + + # put the isos and jigdos in their place + for i in "$OUT"/*-1.iso; do + BASE=`basename $i .iso` + NETINSTNAME=`echo $BASE | sed 's/-BC-1/-businesscard/g;s/-NETINST-1/-netinst/g'` + mv $OUT/$BASE.iso $ARCHDIR/iso-cd/$NETINSTNAME.iso + cat $OUT/$BASE.jigdo | \ + sed 's/-BC-1/-businesscard/g;s/-NETINST-1/-netinst/g' | \ + gzip --best --no-name > $ARCHDIR/jigdo-cd/$NETINSTNAME.jigdo + rm $OUT/$BASE.jigdo + mv $OUT/$BASE.template $ARCHDIR/jigdo-cd/$NETINSTNAME.template + # Make sure that the ISO is as new/newer than the jigdo file; #587774 + touch $ARCHDIR/iso-cd/$NETINSTNAME.iso + done + rm -f "$OUT"/* + END=`now` + mkdir -p "$PUBDIRJIG"/$OUTARCH + echo -e "start=$START\nend=$END\nerror=0\n" > "$PUBDIRJIG"/$OUTARCH/$TRACEFILE else ####################################### @@ -175,7 +202,11 @@ if ./build.sh "$ARCH" >>"$LOGFILE" 2>&1 ; then done # Make sure that the ISOs are as new/newer than the jigdo files; #587774 - touch "$OUT"/*.iso + for file in "$OUT"/*.iso; do + if [ -e $file ] ; then + touch $file + fi + done mkdir -p "$PUBDIRJIG"/$OUTARCH/jigdo-${OUT_TYPE} for file in "$OUT"/*SUMS${SUMS_EXTENSION}; do @@ -192,8 +223,8 @@ if ./build.sh "$ARCH" >>"$LOGFILE" 2>&1 ; then done mv "$OUT"/*.iso "$PUBDIRJIG"/$OUTARCH/iso-${OUT_TYPE} - DATE=`now` - echo -e "end=$DATE\nerror=0\n" > "$PUBDIRJIG"/$OUTARCH/$TRACEFILE + END=`now` + echo -e "start=$START\nend=$END\nerror=0\n" > "$PUBDIRJIG"/$OUTARCH/$TRACEFILE fi touch $TDIR/$CODENAME/1.pkgs_extracted # so we don't get errors below @@ -230,8 +261,8 @@ else mail -s "testingcds $RUN has failed; log included" "$i" done overview "$RUN" failed - DATE=`now` + END=`now` mkdir -p "$PUBDIRJIG"/$OUTARCH - echo -e "end=$DATE\nerror=1\n" > "$PUBDIRJIG"/$OUTARCH/$TRACEFILE + echo -e "start=$START\nend=$END\nerror=0\n" > "$PUBDIRJIG"/$OUTARCH/$TRACEFILE exit 1 fi