extra scripts in use for regular daily and weekly builds

This commit is contained in:
Steve McIntyre 2010-11-14 15:13:48 +00:00
parent 693b4260f5
commit 081cca8e99
8 changed files with 615 additions and 0 deletions

305
contrib/CONF.sh Normal file
View File

@ -0,0 +1,305 @@
#
# This file will have to be sourced where needed
#
# Unset all optional variables first to start from a clean state
unset NONUS || true
unset FORCENONUSONCD1 || true
unset NONFREE || true
unset CONTRIB || true
unset EXTRANONFREE || true
unset LOCAL || true
unset LOCALDEBS || true
unset SECURED || true
unset SECURITY || true
unset BOOTDIR || true
unset BOOTDISKS || true
unset SYMLINK || true
unset COPYLINK || true
unset MKISOFS || true
unset MKISOFS_OPTS || true
unset ISOLINUX || true
unset EXCLUDE || true
unset SRCEXCLUDE || true
unset NORECOMMENDS || true
unset NOSUGGESTS || true
unset IMAGESUMS || true
unset JIGDOCMD || true
unset JIGDOTEMPLATEURL || true
#unset JIGDOFALLBACKURLS || true
unset JIGDOINCLUDEURLS || true
unset JIGDOSCRIPT || true
unset JIGDO_OPTS || true
#unset DEFBINSIZE || true
#unset DEFSRCSIZE || true
unset FASTSUMS || true
unset PUBLISH_URL || true
unset PUBLISH_NONUS_URL || true
unset PUBLISH_PATH || true
unset UDEB_INCLUDE || true
unset UDEB_EXCLUDE || true
unset BASE_INCLUDE || true
unset BASE_EXCLUDE || true
#unset INSTALLER_CD || true
# The debian-cd dir
# Where I am (hoping I'm in the debian-cd dir)
export BASEDIR=`pwd`
# Building squeeze cd set ...
export CODENAME=squeeze
if [ ! "$DI_CODENAME" ]
then
export DI_CODENAME=$CODENAME
fi
# Version number, "2.2 r0", "2.2 r1" etc.
#export DEBVERSION="Lenny-DI-rc2"
export DEBVERSION="testing"
# Official or non-official set.
# NOTE: THE "OFFICIAL" DESIGNATION IS ONLY ALLOWED FOR IMAGES AVAILABLE
# ON THE OFFICIAL DEBIAN CD WEBSITE http://cdimage.debian.org
#export OFFICIAL="Unofficial"
export OFFICIAL="Official Snapshot"
#export OFFICIAL="Official RC"
#export OFFICIAL="Official"
# ... for arch
if [ ! "$ARCH" ]
then
export ARCH=`dpkg --print-installation-architecture`
fi
# IMPORTANT : The 4 following paths must be on the same partition/device.
# If they aren't then you must set COPYLINK below to 1. This
# takes a lot of extra room to create the sandbox for the ISO
# images, however. Also, if you are using an NFS partition for
# some part of this, you must use this option.
# Paths to the mirrors
if [ "$MIRROR"x = ""x ] ; then
export MIRROR=/org/cdbuilder.debian.org/src/ftp/debian
fi
NUM_ARCHES=`echo $ARCH | wc -w`
if [ "$NUM_ARCHES"x = "1"x ] ; then
OUTARCH=$ARCH
else
OUTARCH=multi-arch
fi
# Path of the temporary directory
export TDIR=/org/cdbuilder.debian.org/src/deb-cd/tmp/"$INSTALLER_CD""$DI""$OUTARCH"
# Path where the images will be written
if [ "$OUT"x = ""x ] ; then
export OUT=/org/cdbuilder.debian.org/dst/deb-cd/out/"$INSTALLER_CD""$DI""$OUTARCH"
fi
# Where we keep the temporary apt stuff.
# This cannot reside on an NFS mount.
export APTTMP=$TDIR/apt
# Do I want to have NONFREE merged in the CD set
# export NONFREE=1
# Do I want to have CONTRIB merged in the CD set
export CONTRIB=1
# Do I want to have NONFREE on a separate CD (the last CD of the CD set)
# WARNING: Don't use NONFREE and EXTRANONFREE at the same time !
# export EXTRANONFREE=1
# If you have a $MIRROR/dists/$CODENAME/local/binary-$ARCH dir with
# local packages that you want to put on the CD set then
# uncomment the following line
# export LOCAL=1
# If your local packages are not under $MIRROR, but somewhere else,
# you can uncomment this line and edit to to point to a directory
# containing dists/$CODENAME/local/binary-$ARCH
# export LOCALDEBS=/home/joey/debian/va/debian
# If you want a <codename>-secured tree with a copy of the signed
# Release.gpg and files listed by this Release file, then
# uncomment this line
# export SECURED=1
# Where to find the security patches. This directory should be the
# top directory of a security.debian.org mirror.
#export SECURITY="$TOPDIR"/debian/debian-security
# Sparc only : bootdir (location of cd.b and second.b)
# export BOOTDIR=/boot
# Symlink farmers should uncomment this line :
# export SYMLINK=1
# Use this to force copying the files instead of symlinking or hardlinking
# them. This is useful if your destination directories are on a different
# partition than your source files.
# export COPYLINK=1
# Options
#export MKISOFS="$BASEDIR/../mkisofs/usr/bin/mkisofs"
#export MKISOFS="$BASEDIR/../genisoimage/usr/bin/genisoimage"
export MKISOFS="$BASEDIR/../genisoimage"
#export MKISOFS_OPTS="-jigdo-template-compress bzip2 -r -checksum_algorithm_iso md5,sha1,sha256,sha512"
export MKISOFS_OPTS="-jigdo-template-compress bzip2 -r " #-checksum_algorithm_iso md5,sha1"
# export MKISOFS_OPTS="-r" #For normal users
# export MKISOFS_OPTS="-r -F ." #For symlink farmers
# ISOLinux support for multiboot on CD1 for i386
export ISOLINUX=1
# uncomment this to if you want to see more of what the Makefile is doing
export VERBOSE_MAKE=1
# The maximum size allowed for an individual package, in bytes; if
# larger than this, it will be excluded (and all dependents, of
# course)
#export MAX_PKG_SIZE=600000000
# uncoment this to make build_all.sh try to build a simple CD image if
# the proper official CD run does not work
#ATTEMPT_FALLBACK=yes
# Set your disk size here in MB. Used in calculating package and
# source file layouts in build.sh and build_all.sh. Defaults are for
# CD-R, try ~4600 for DVD-R.
if [ ! "$DEFBINSIZE" ] ; then export DEFBINSIZE=642 ; fi
if [ ! "$DEFSRCSIZE" ] ; then export DEFSRCSIZE=642 ; fi
if [ "$DISKTYPE"x = ""x ] ; then
DISKTYPE=CD
fi
export DISKTYPE
# List of languages for which language tasks from tasksel should be
# included. See tasks/README.tasksel for further info.
export TASK_LANGLIST=tasksel_d-i.languages
# We don't want certain packages to take up space on CD1...
#export EXCLUDE1=exclude
# ...but they are okay for other CDs (UNEXCLUDEx == will be included on CD x if not already covered)
#export UNEXCLUDE2=unexclude-CD2
# Any packages listed in EXCLUDE but not in any UNEXCLUDE will be
# excluded completely.
# We also exclude some source packages
#export SRCEXCLUDE=exclude-src
# Set this if the recommended packages should be skipped when adding
# package on the CD. The default is 'false'.
export NORECOMMENDS=1
# Set this if the suggested packages should be skipped when adding
# package on the CD. The default is 'true'.
#export NOSUGGESTS=1
# Set to 1 to generate MD5 and SHA1 sums for generated images
export IMAGESUMS=1
# We may have to extract files from packages to put them onto the CD
# (e.g. bootloader files). If you make those packages (and their
# sources) available somewhere, list it here so that README.source
# can point to it
export ARCHIVE_EXTRACTED_SOURCES="http://cdimage.debian.org/cdimage/cd-sources/"
# Produce iso/jigdo files: specify how many iso/jigdo files should be
# produced in your set. If not set or when the value is "ALL" they will
# be created for all images. One of the variables can be set to zero if
# either iso or jigdo files are not wanted, but not both.
# Replaces the old "DOJIGDO" setting with something much more flexible.
#export MAXISOS=0
#export MAXJIGDOS=0
# Space-separated list of "include URLs" to add to the .jigdo file.
# The included files are used to provide an up-to-date list of Debian
# mirrors to the jigdo _GUI_application_ (_jigdo-lite_ doesn't support
# "[Include ...]").
export JIGDOINCLUDEURLS="http://cdimage.debian.org/debian-cd/debian-servers.jigdo"
#
# $JIGDOTEMPLATEURL and $JIGDOINCLUDEURLS are passed to
# "tools/jigdo_header", which is used by default to generate the
# [Image] and [Servers] sections of the .jigdo file. You can provide
# your own script if you need the .jigdo file to contain different
# data.
#export JIGDOSCRIPT="myscript"
# If set, use the md5sums from the main archive, rather than calculating
# them locally
export FASTSUMS=1
# A couple of things used only by publish_cds, so it can tweak the
# jigdo files, and knows where to put the results.
# You need to run publish_cds manually, it is not run by the Makefile.
#export PUBLISH_URL="http://cdimage.debian.org/jigdo-area"
#export PUBLISH_NONUS_URL="http://non-US.cdimage.debian.org/jigdo-area"
#export PUBLISH_PATH="/home/jigdo-area/"
# Specify files and directories to *exclude* from jigdo processing. These
# files on each CD are expected to be different to those on the mirror, or
# are often subject to change. Any files matching entries in this list will
# simply be placed straight into the template file.
export JIGDO_EXCLUDE="'README*' /doc/ /md5sum.txt /.disk/ /pics/ 'Release*' 'Packages*' 'Sources*'"
# Specify the minimum file size to consider for jigdo processing. Any files
# smaller than this will simply be placed straight into the template file.
export JIGDO_OPTS="-jigdo-min-file-size 0"
for EXCL in $JIGDO_EXCLUDE
do
JIGDO_OPTS="$JIGDO_OPTS -jigdo-exclude $EXCL"
done
export IGNORE_MISSING_BOOT_SCRIPT=1
# Where to find the boot disks
#export BOOTDISKS=$TOPDIR/ftp/skolelinux/boot-floppies
# File with list of packages to include when fetching modules for the
# first stage installer (debian-installer). One package per line.
# Lines starting with '#' are comments. The package order is
# important, as the packages will be installed in the given order.
#export UDEB_INCLUDE="$BASEDIR"/data/$CODENAME/udeb_include
# File with list of packages to exclude as above.
#export UDEB_EXCLUDE="$BASEDIR"/data/$CODENAME/udeb_exclude
# File with list of packages to include when running debootstrap from
# the first stage installer (currently only supported in
# debian-installer). One package per line. Lines starting with '#'
# are comments. The package order is important, as the packages will
# be installed in the given order.
#export BASE_INCLUDE="$BASEDIR"/data/$CODENAME/base_include
# File with list of packages to exclude as above.
#export BASE_EXCLUDE="$BASEDIR"/data/$CODENAME/base_exclude
# Only put the installer onto the cd (set NORECOMMENDS,... as well).
# INSTALLER_CD=0: nothing special (default)
# INSTALLER_CD=1: just add debian-installer (use TASK=debian-installer)
# INSTALLER_CD=2: add d-i and base (use TASK=debian-installer+kernel)
#export INSTALLER_CD=0
# Set to 1 to save space by omitting the release notes
# If so we will link to them on the web site.
export OMIT_RELEASE_NOTES=1
# Set this to override the defaul location
#export RELEASE_NOTES_LOCATION="http://www.debian.org/releases/$CODENAME"
case "$OFFICIAL"x in
"Official"x)
export OFFICIAL_VAL=2
;;
"Official Beta"x|"Official Snapshot"x)
export OFFICIAL_VAL=1
;;
*)
export OFFICIAL_VAL=0
;;
esac

69
contrib/common.sh Normal file
View File

@ -0,0 +1,69 @@
# Common handy shell script functions
now () {
date -u +%F:%H:%M:%S
}
calc_time () {
echo $1 $2 | awk '
{
split($1, start, ":")
start_time = (3600*start[2]) + (60*start[3]) + start[4]
split($2, end, ":")
end_time = (3600*end[2]) + (60*end[3]) + end[4]
# Cope with going to a new day; do not worry about more than 1 day!
if (start[1] != end[1]) { end_time += 86400 }
time_taken = end_time - start_time
hours = int(time_taken / 3600)
time_taken -= (hours * 3600)
minutes = int(time_taken / 60)
time_taken -= (minutes * 60)
seconds = time_taken
printf("%dh%2.2dm%2.2ds\n", hours, minutes, seconds)
}'
}
build_started () {
export BUILDNAME=$1
BUILDS_RUNNING="$BUILDS_RUNNING $BUILDNAME"
export ${BUILDNAME}START=`now`
}
build_finished () {
ARCH="$1"
BUILDNAME="$2"
BUILDNAMESTART="${BUILDNAME}START"
start=${!BUILDNAMESTART}
. $PUBDIRJIG/$ARCH/$BUILDNAME-trace
time_spent=`calc_time $start $end`
echo " $ARCH $BUILDNAME build started at $start, ended at $end (took $time_spent), error $error"
if [ $error -ne 0 ] ; then
arch_error="$arch_error "$BUILDNAME"FAIL/$error/$end"
fi
}
catch_parallel_builds () {
# Catch parallel builds here
while [ "$BUILDS_RUNNING"x != ""x ] ; do
BUILDS_STILL_RUNNING=""
for BUILDNAME in $BUILDS_RUNNING; do
if [ -e $PUBDIRJIG/$arch/$BUILDNAME-trace ] ; then
build_finished $arch $BUILDNAME
else
BUILDS_STILL_RUNNING="$BUILDS_STILL_RUNNING $BUILDNAME"
fi
done
BUILDS_RUNNING=$BUILDS_STILL_RUNNING
if [ "$BUILDS_RUNNING"x != ""x ] ; then
sleep 1
fi
done
if [ "$arch_error"x = ""x ] ; then
arch_error="none"
fi
arch_end=`now`
arch_time=`calc_time $arch_start $arch_end`
echo "$ARCH build started at $arch_start, ended at $arch_end (took $arch_time), error(s) $arch_error"
}

View File

@ -0,0 +1,39 @@
#!/bin/sh
set -e
OUT=/mnt/nfs-cdimage/unofficial/non-free/firmware
TOPDIR=~/build
SUITE=$1
. $TOPDIR/CONF.sh
if [ "$SUITE"x = ""x ] ; then
echo "Need to know what to produce!"
exit 1
fi
# Do stuff here!
export TDIR=/org/cdbuilder.debian.org/dst/deb-cd/tmp/firmware
export BASEDIR=$TOPDIR/debian-cd
mkdir -p $TDIR
fakeroot $BASEDIR/tools/make-firmware-image $MIRROR $SUITE $TDIR $BASEDIR
DATE=`date +%Y%m%d`
mkdir -p $OUT/$SUITE/$DATE
mv $TDIR/firmware*.* $OUT/$SUITE/$DATE/
# Update the current link, and delete the old surplus builds. Keep up
# to 3
cd $OUT/$SUITE
rm -f current
ln -sf $DATE current
echo "Removing old firmware directories:"
NUM=`ls -d 20*/ 2>/dev/null |wc -l`
NUM=$(($NUM-6))
if [ "$NUM" -gt "0" ] ; then
REMOVE=`ls -1d 20* 2>/dev/null|head -n $NUM`
echo " $REMOVE"
rm -rf $REMOVE
fi

93
contrib/iso_run Executable file
View File

@ -0,0 +1,93 @@
#!/bin/sh
#
# Helper for image creation
#
PUBDIRJIG=$1
RSYNC_TARGET=$2
ARCH=$3
LOG=~/build/log/$ARCH.iso_run
HOSTNAME=`hostname -f`
LOCK=~/iso_run.lock
START=`date -u +%H:%M:%S`
# Check to see if another sync is in progress
if lockfile -! -l 43200 -r-1 "$LOCK"; then
echo $HOSTNAME is not doing another iso_run, lock file $LOCK exists
exit 1
fi
trap "rm -f $LOCK > /dev/null 2>&1" exit
rm -f $LOG
# Given an ISO image:
# 1. create the torrent file
# 2. create the list file
# 3. copy all of them into place
process_iso() {
FILE=$1
OUTDIR=$2
BTFILE=`echo $FILE.torrent | sed 's/iso-/bt-/'`
~/build/mktorrent $FILE >> $LOG
LISTFILE=`echo $FILE | sed 's/iso-/list-/g;s/\.iso$/.list.gz/g'`
~/build/mklist $FILE >> $LOG
echo $OUTDIR/$FILE >> $LOG
cp -a $FILE $OUTDIR/$FILE
echo $OUTDIR/$BTFILE >> $LOG
cp -a $BTFILE $OUTDIR/$BTFILE
echo $OUTDIR/$LISTFILE >> $LOG
cp -a $LISTFILE $OUTDIR/$LISTFILE
}
# Poor man's rsync, but with some local optimisations
copy_files() {
SRC=$1
TARGET=$2
ARCHES=$3
CURRENT=`pwd`
cd $SRC
for ARCH in $ARCHES
do
for DISKTYPE in dvd cd bd; do
for DIRTYPE in bt iso jigdo list; do
mkdir -p -m775 $TARGET/$ARCH.tmp/$DIRTYPE-$DISKTYPE
done
done
cd $ARCH
find . -name '*.jigdo' -o -name '*.template' -o -name '*SUMS*' | \
xargs tar cf - | (cd $TARGET/$ARCH.tmp/ && tar xvf -) >> $LOG
rm -rf bt-* list-*
mkdir bt-cd bt-dvd bt-bd list-cd list-dvd list-bd
for file in iso-*/*.iso; do
if [ -e $file ] ; then
process_iso $file $TARGET/$ARCH.tmp
fi
done
DATE=`date -u`
sed "s/ARCH/$ARCH/g;s/DATE/$DATE/g" ~/build/weekly.html \
> $TARGET/$ARCH.tmp/HEADER.html
cd ..
done
cd $CURRENT
}
copy_files $PUBDIRJIG $RSYNC_TARGET $ARCH
echo "$START: Starting $ARCH sync from $PUBDIRJIG to $RSYNC_TARGET" >> $LOG
mv $RSYNC_TARGET/$ARCH $RSYNC_TARGET/$ARCH.old
mv $RSYNC_TARGET/$ARCH.tmp $RSYNC_TARGET/$ARCH
rm -rf $RSYNC_TARGET/$ARCH.old &
# Update the trace file now to force a sync on free.hands.com after each arch
mkdir -p $RSYNC_TARGET/trace
date -u > $RSYNC_TARGET/trace/cdimage.debian.org
END=`date -u +%H:%M:%S`
echo "$ARCH synced across; started at $START, ended at $END"
echo "$END: Finished" >> $LOG

40
contrib/merge-sums Executable file
View File

@ -0,0 +1,40 @@
#!/bin/sh
export LANG=C
export LC_ALL=C
if [ "$1"x = ""x ] ; then
echo "$0: tell me which dir to target!"
echo "abort"
exit 1
fi
CHECK=`ls -al $1/*/iso-* 2>/dev/null`
if [ "$CHECK"x = ""x ] ; then
echo "$0: could not find stuff to work on in $1"
echo "abort"
exit 1
fi
for DIR in $1/*/*-*; do
cd $DIR
CHECK=`ls -al *SUMS.* 2>/dev/null`
if [ "$CHECK"x != ""x ] ; then
cat MD5SUMS.* | sort -k2 | uniq > MD5SUMS
cat SHA1SUMS.* | sort -k2 | uniq > SHA1SUMS
cat SHA256SUMS.* | sort -k2 | uniq > SHA256SUMS
cat SHA512SUMS.* | sort -k2 | uniq > SHA512SUMS
for ISOFILE in `cat MD5SUMS SHA1SUMS | awk '{print $2}'`; do
JIGDOFILE=${ISOFILE%iso}jigdo
TEMPLATEFILE=${ISOFILE%iso}template
if [ ! -e $ISOFILE ] && [ ! -e $JIGDOFILE ] && [ ! -e $TEMPLATEFILE ] ; then
echo "$ISOFILE/$JIGDOFILE/$TEMPLATEFILE missing in $DIR!"
# echo "abort"
# exit 1
fi
done
echo "$DIR done"
else
echo "Ignoring $DIR: no checksum files"
fi
done

17
contrib/mklist Executable file
View File

@ -0,0 +1,17 @@
#!/bin/sh
for FILE in $@; do
if [ -e "$FILE" ] ; then
ISODIR=`dirname $FILE`
LISTDIR=`echo $ISODIR | sed 's/iso-/list-/'`
if [ ! -d $LISTDIR ] ; then
mkdir -p $LISTDIR
fi
LISTFILE=`echo $FILE | sed 's/\.iso$/.list.gz/g'`
isoinfo -fR -i $FILE | grep /pool/ | \
awk '/\.deb$|\.udeb$|\.tar\.gz$|\.dsc$|\.diff\.gz$/ {gsub(".*/","");print $1}' | \
LANG=C sort | gzip -9 > $LISTFILE
mv $LISTFILE $LISTDIR
fi
done

14
contrib/mktorrent Executable file
View File

@ -0,0 +1,14 @@
#!/bin/sh
for FILE in $@; do
MKTORRENT=/home/debian-cd/bt/btmakemetafile.py
ISODIR=`dirname $FILE`
BTDIR=`echo $ISODIR | sed 's/iso-/bt-/;s/usb-/bt-/'`
if [ ! -d $BTDIR ] ; then
mkdir -p $BTDIR
fi
$MKTORRENT http://bttracker.debian.org:6969/announce \
--comment '"Debian CD from cdimage.debian.org"' \
$FILE | grep -v complete
mv $FILE.torrent $BTDIR
done

38
contrib/report_build_error Executable file
View File

@ -0,0 +1,38 @@
#!/bin/sh
OUTDIR=$1
ARCH=$2
RESULTS=$3
OUTFILE=$OUTDIR/$ARCH/HEADER.html
grep -q WARNING $OUTFILE
if [ $? -ne 0 ] ; then
echo >> $OUTFILE
echo "<p><strong>WARNING:</strong>" >> $OUTFILE
echo "This build is not up to date; it is the most recent successful build.</p>" >> $OUTFILE
echo >> $OUTFILE
echo "<p>Later weekly builds failed with errors:</p>" >> $OUTFILE
fi
echo "<hr>" >> $OUTFILE
echo "<ul>" >> $OUTFILE
for RESULT in $RESULTS; do
TYPE=`echo $RESULT | awk -F / '{print $1}'`
ERROR=`echo $RESULT | awk -F / '{print $2}'`
DATE=`echo $RESULT | awk -F / '{print $3}'`
case $TYPE in
CDFAIL)
echo "<li>Full CD build failed with error $ERROR at $DATE" >> $OUTFILE ;;
DVDFAIL)
echo "<li>DVD build failed with error $ERROR at $DATE" >> $OUTFILE ;;
BDFAIL)
echo "<li>Blu-ray build failed with error $ERROR at $DATE" >> $OUTFILE ;;
KDECDFAIL)
echo "<li>KDE CD build failed with error $ERROR at $DATE" >> $OUTFILE ;;
LIGHTCDFAIL)
echo "<li>XFCE/lxde CD build failed with error $ERROR at $DATE" >> $OUTFILE ;;
esac
done
echo "</ul>" >> $OUTFILE