diff --git a/contrib/jigdo-watcher b/contrib/jigdo-watcher new file mode 100755 index 00000000..70e92a9d --- /dev/null +++ b/contrib/jigdo-watcher @@ -0,0 +1,55 @@ +#!/bin/bash + +STATE=/home/stevem/watch-state + +R_HOST=cd-builder.debian.net +R_LOC=testing/weekly-builds +R_WEBROOT=$R_LOC +R_RSYNCROOT=/org/cdimage.debian.org/www/$R_LOC +R_USER=steve + +CACHE_DIR=/org/jigdo-snapshots +OUT_DIR=/org/jigdo-area/testing +MIRROR=/org/ftp/debian +SNAP_AREA=/org/jigdo-area/snapshot +LOCK=$STATE/lock + +# Check to see if another sync is in progress +if lockfile -! -l 43200 -r 0 "$LOCK" >/dev/null 2>&1 ; then +# echo `hostname` is unable to start CD sync, lock file exists + exit 1 +fi +trap "rm -f $LOCK > /dev/null 2>&1" exit + +wget -q -O $STATE/cd.new http://$R_HOST/$R_WEBROOT/trace/bla.wolffelaar.nl +OLD=`cat $STATE/cd` +NEW=`cat $STATE/cd.new` +if [ "$OLD"x != "$NEW"x ] ; then + echo "Old date $OLD, New $NEW" + echo "Time to get new jigdos" + mv -f $STATE/cd.new $STATE/cd + + rsync -rvtlz --delete \ + $R_USER@$R_HOST:$R_RSYNCROOT/ \ + $OUT_DIR/ + + SNAP_DATE=`date +%Y%m%d` + echo "And snapshot to $SNAP_DATE" + + mkdir -p $CACHE_DIR/$SNAP_DATE + cd $OUT_DIR + find . -name '*.jigdo' | xargs tar cf - | ( cd $CACHE_DIR/$SNAP_DATE && tar xf - ) + + # Now generate a list of all the jigdo files that we should be + # looking at + find $CACHE_DIR -name '*.jigdo' > $CACHE_DIR/jigdo.list + + # And now update and check the snapshot from those jigdo files + DATE=`date` + echo "$DATE: Updating snapshot" + + ~/bin/mkjigsnap -m $MIRROR -d $SNAP_AREA/Debian -j $CACHE_DIR/jigdo.list -t ~/tmp + + DATE=`date` + echo " $DATE: done" +fi diff --git a/contrib/mkjigsnap b/contrib/mkjigsnap new file mode 100755 index 00000000..3972ac40 --- /dev/null +++ b/contrib/mkjigsnap @@ -0,0 +1,226 @@ +#!/bin/sh +# +# mkjigsnap +# +# (c) 2004 Steve McIntyre +# +# Server-side wrapper; run this on a machine with a mirror to set up +# the snapshots for jigit +# +# GPL v2 - see COPYING +# +# Some things needed: +# location of the mirror +# the keyword to look for (e.g. Debian) +# the snapshot dirname (e.g. today's date) +# the jigdo files +# Example: +# ./mkjigsnap -m /tmp/mirror \ +# -k Debian -k Non-US \ +# -d 20041017 \ +# jigdo1 jigdo2 jigdo3 ... + +STARTDATE=`date` + +while [ $# -gt 0 ] +do + case "$1"x in + "-m"x) + shift + MIRROR=$1 + shift + ;; + "-d"x) + shift + DIRNAME=$1 + shift + ;; + "-j"x) + shift + JIGDOLIST=$1 + shift + ;; + "-t"x) + shift + TF_TMPDIR="--directory $1" + SORT_TMPDIR="-T $1" + shift + ;; + *) + JIGDOS="$JIGDOS $1" + shift + ;; + esac +done + +if [ "$MIRROR"x = ""x ] ; then + echo "You must specify the location of the mirror!" + exit 1 +fi + +if [ "$DIRNAME"x = ""x ] ; then + echo "You must specify the snapshot directory name!" + exit 1 +fi + +if [ "$JIGDOS"x = ""x ] ; then + if [ "$JIGDOLIST"x = ""x ] ; then + echo "You must specify some jigdo files!" + exit 1 + fi +fi + +TMPFILE=`tempfile $TF_TMPDIR` +if [ "$JIGDOLIST"x != ""x ] ; then + for JIGDO in `cat $JIGDOLIST` + do + zcat -f $JIGDO | sed -n "s/^.*Debian://gp" >> $TMPFILE + done +fi + +if [ "$JIGDOS"x != ""x ] ; then + zcat -f $JIGDOS | sed -n "s/^.*Debian://gp" >> $TMPFILE +fi + +LISTDONEDATE=`date` + +TMPFILE1=`tempfile $TF_TMPDIR` +TOTAL_FILES=`wc -l < $TMPFILE` + +cat $TMPFILE | sort -u $SORT_TMPDIR > $TMPFILE1 + +SORTDONEDATE=`date` +NUM_FILES=`wc -l < $TMPFILE1` + +mv -f $TMPFILE1 $TMPFILE + +NUM=`cat $TMPFILE | wc -l` + +LINKS_DONE=0 +(echo $NUM; echo $MIRROR; echo $DIRNAME; cat $TMPFILE ) | perl -we ' + + use File::Basename; + use File::Find; + my $num; + my $mirrorpath; + my $outdir; + my $dirname; + my $filename; + my $done = 0; + my $failed = 0; + my @file_list; + my $old_deleted = 0; + my $link; + $| = 1; + + # Make a dir tree + sub mkdirs { + my $input = shift; + my $dir; + my @components; + my $need_slash = 0; + + if (! -d $input) { + @components = split /\//,$input; + foreach $component (@components) { + if ($need_slash) { + $dir = join ("/", $dir, $component); + } else { + $dir = $component; + $need_slash = 1; + } + if (! -d $dir) { + mkdir $dir; + } + } + } + } + + sub delete_redundant { + my $ref; + + if (-f) { + $ref = $file_list{$File::Find::name}; + if (!defined($ref)) { + unlink($File::Find::name); + $old_deleted++; + if ( !($old_deleted % 1000) ) { + print "$old_deleted\n"; + } + } + } + } + + while (<>) { + chomp; + + if (!defined($num)) { + $num = $_; + next; + } + if (!defined($mirrorpath)) { + $mirrorpath = $_; + next; + } + if (!defined($outdir)) { + $outdir = $_; + print "Linking $num files from $mirrorpath to $outdir\n"; + next; + } + + $outfile = $outdir . "/" . $_; + $file_list{$outfile}++; + if (! -e $outfile) { + $dirname = dirname($_); + $filename = basename($_); + + mkdirs($outdir . "/" . $dirname); + my $link_ok = 0; + foreach $mirror (split /:/,$mirrorpath) { + $infile = $mirror . "/" . $_; + if (-l $infile) { + $link = readlink($infile); + if ($link =~ m#^/#) { + $infile = $link; + } else { + $infile = dirname($infile) . "/" . $link; + } + } + $outfile = $outdir . "/" . $_; + if (link ($infile, $outfile)) { + $link_ok = 1; + last; + } + $infile = $mirror . "/" . $filename; +# print "Falling back to $infile\n"; + if (link ($infile, $outfile)) { + $link_ok = 1; + last; + } + } + if ($link_ok == 0) { + print "\nFailed to create link $outfile\n"; + $failed++; + } + } + $done++; + if ( !($done % 1000) ) { + print "$done/$num\n"; + } + } + print " Finished: $done/$num, $failed failed\n\n"; + + # Now walk the tree and delete files that we no longer need + print "Scanning for now-redundant files\n"; + find(\&delete_redundant, $outdir); + print " Finished: $old_deleted old files removed\n"; +' + +rm -f $TMPFILE +echo + +echo "$STARTDATE: startup" +echo "$LISTDONEDATE: $TOTAL_FILES found" +echo "$SORTDONEDATE: $NUM_FILES after sorting" +echo `date`": mkjigsnap finished successfully" + diff --git a/contrib/testingcds b/contrib/testingcds new file mode 100644 index 00000000..9521a134 --- /dev/null +++ b/contrib/testingcds @@ -0,0 +1,258 @@ +#!/bin/bash +# +# testingcds (c) 2005 Santiago Garcia Mantinan +# +# GPL v2 +# +# This file is included here just as an example, it was used when building the +# official testing cds at some point, it was written for a very specific setup +# and then went over a lot of changes during its life, so now this code is +# horrible, hoever some fellows feel it should be included in debian-cd, +# well, I (manty) will leave that on their hands. +# +# The setup we have uses normal debian-cd for daily builds, while it uses a jte +# patched debian-cd for weekly builds, this is done just for speed, the script +# should work with normal debian-cd for all kind of builds just removing -jte +# down there. We are currently just building isos for dailies and jigdos for +# weeklies, no snapshots being generated on the build machine, so I cannot +# waranty that all the code down here works, again, I must stress that you +# should take this just as an example, because part of it is untested and +# the rest is written for a very very specific setup, so... no waranties at all +# +# This is how this script is called for the daily images: +# OMIT_MANUAL=1 NORECOMMENDS=1 NOSUGGESTS=1 COMPLETE=0 INSTALLER_CD=1 TASK=tasks/debian-installer-sarge DOJIGDO=0 DI=sarge DI_DIST=sarge ./testingcds "$arch" +# OMIT_MANUAL=1 NORECOMMENDS=1 NOSUGGESTS=1 COMPLETE=0 INSTALLER_CD=2 TASK=tasks/debian-installer+kernel-sarge DOJIGDO=0 DI=sarge DI_DIST=sarge ./testingcds "$arch" +# OMIT_MANUAL=1 NORECOMMENDS=1 NOSUGGESTS=1 COMPLETE=0 INSTALLER_CD=1 TASK=tasks/debian-installer-sid DOJIGDO=0 DI=sid DI_WWW_HOME=default DI_DIR="$ARCH_DI_DIR" ./testingcds "$arch" +# OMIT_MANUAL=1 NORECOMMENDS=1 NOSUGGESTS=1 COMPLETE=0 INSTALLER_CD=2 TASK=tasks/debian-installer+kernel-sid DOJIGDO=0 DI=sid DI_WWW_HOME=default DI_DIR="$ARCH_DI_DIR" ./testingcds "$arch" +# This is respectively for sarge_d-i businesscards, sarge_d-i netinsts, +# sid_d-i businesscards, and sid_d-i netinsts. +# Where $ARCH_DI_DIR is set to the path for the local mirror of the d-i daily +# built images, this parameter can be omited if no mirror of the images is used + +# For the weekly images it is run for normal cds like this: +# DI_DIST=sarge ./testingcds "$arch" +# and for the dvds like this: +# PUBDIRJIG=/path_to_the/dvd INSTALLER_CD=3 DEFBINSIZE=4440 DEFSRCSIZE=4432 DI_DIST=sarge ./testingcds "$arch" + + +# Set up and cleaning +if [ $# -lt 1 ] +then + echo $0 \$ARCH + echo "variables: PUBDIR[ISO|JIG]" + exit 1 +fi + +if [ ! "$DATE" ];then DATE=`/bin/date -u +%Y%m%d`;fi + +export ARCH=$1 +export PATH=$PATH:/sbin:/usr/sbin +if [ ! "$PUBDIRJIG" ];then PUBDIRJIG=/org/cdimage.debian.org/www/testing/cd;fi +if [ ! "$PUBDIRISO" ];then PUBDIRISO=/org/cdimage.debian.org/www/testing;fi +if [ ! "$JIGDOFALLBACKURLS" ];then + if [ "$ARCH" = "amd64" ];then + export JIGDOFALLBACKURLS="Debian=http://amd64-cdsnap.debian.net/cdimage/snapshot-amd64/Debian/" + else + export JIGDOFALLBACKURLS="Debian=http://us.cdimage.debian.org/cdimage/snapshot/Debian/" + fi +fi +# $DI is used to set DI_CODENAME and also to specify the directory for images +if [ "$DI" ];then + export DI_CODENAME="$DI" + PUBDIRJIG="$PUBDIRJIG"/"$DI"_d-i + PUBDIRISO="$PUBDIRISO"/"$DI"_d-i +fi + +CALC="Using previous sizes estimations." +if [ "$INSTALLER_CD" = "1" -o "$INSTALLER_CD" = "2" ] +then + # Do a tradicional build for the ISOs of the small cds + cd /org/cdimage.debian.org/setup/debian-cd && + . ../CONF.sh + # this is for build.sh + export CF=../CONF.sh + export IMAGETARGET=bin-official_images + # to save some time + export SKIPMIRRORCHECK=yes +else + # Do the size stimation and a JTE build for jigdos + cd /org/cdimage.debian.org/setup/debian-cd-jte && + . ../CONF.sh && + export IMAGETARGET="ok bootable upgrade bin-infos bin-list" && + export CF=../CONF.sh && + ./build.sh $ARCH >/dev/null 2>&1 && + cd tools && CALC=`./calc` && cd .. + # this is for build.sh + if [ "$ARCH" = "i386" ] + then + export IMAGETARGET=official_images + else + export IMAGETARGET=bin-official_images + fi +fi + +if [ ! "$OUT" ];then echo Error on the config;exit 1;fi + +if [ "$INSTALLER_CD" = "1" ];then export OFFICIAL="Official BusinessCard Snapshot";fi +if [ "$INSTALLER_CD" = "2" ];then export OFFICIAL="Official NetInst Snapshot";fi + +export LOG="`pwd`/../log/" + +rm -rf "$OUT" +mkdir -p "$OUT" +mkdir -p "$LOG" + +overview_log="$LOG/overview" +overview () { + if [ -e $overview_log ]; then + grep -v " $1 " $overview_log > $overview_log.new + mv $overview_log.new $overview_log + fi + LANG=C echo "$(dpkg --print-architecture) ($(date)) $(whoami)@$(hostname | cut -d . -f 1) $1 $2" >> $overview_log +} + +# Upgrade sarge's debian-installer task file +# changed this old line into the new one so that amd64 doesn't break +# cd tasks && ../tools/generate_di_list && ../tools/generate_di+k_list && cd .. || exit 1 +cd tasks && ../tools/generate_di_list ; ../tools/generate_di+k_list ; cd .. +DEBOOTUDEB=`/bin/sed -n \ + 's/Filename: \(pool\/main\/d\/debootstrap\/debootstrap-udeb.*udeb\)$/\1/p' \ + $MIRROR/dists/$DI_CODENAME/main/debian-installer/binary-$ARCH/Packages` +if [ -z "$DEBOOTUDEB" ];then exit 1;fi +echo "$CALC" >"$LOG/$INSTALLER_CD$DI$ARCH" +echo Using ${DEBOOTUDEB##*/} to get packages used by d-i. >>"$LOG/$INSTALLER_CD$DI$ARCH" +rm -rf ../debootstrap +dpkg -x "$MIRROR/$DEBOOTUDEB" ../debootstrap + +# Do the actual building +if ./build.sh $ARCH >>"$LOG/$INSTALLER_CD$DI$ARCH" 2>&1;then + +# if it went ok get things ready for publishing + +# we treat the netinst images in a different way + +if [ "$INSTALLER_CD" = "1" -o "$INSTALLER_CD" = "2" ] +then + +# NETINST STUFF (ISO ONLY) + +# make the dirs we'll use to publish things + mkdir -p "$OUT"/$ARCH +# rename the isos from .raw to .iso and put them in their place + for i in "$OUT"/*-1.raw + do + I=${i##*/} + if [ "$INSTALLER_CD" = "1" ];then + NETINSTNAME=${I%-binary-1.raw}-businesscard.iso + else + NETINSTNAME=${I%-binary-1.raw}-netinst.iso + fi + mv $i "$OUT"/$ARCH/$NETINSTNAME + done +# change images names from raw to iso on the MD5SUMS + sed "s/[^ ]*-1.raw$/$NETINSTNAME/" "$OUT"/MD5SUMS >>"$OUT"/$ARCH/MD5SUMS +# Publish the new version + mkdir -p "$PUBDIRISO"/$ARCH/$DATE + if [ -e "$PUBDIRISO"/$ARCH/$DATE/MD5SUMS ];then + grep -v $NETINSTNAME "$PUBDIRISO"/$ARCH/$DATE/MD5SUMS > "$PUBDIRISO"/$ARCH/$DATE/MD5SUMS.tmp + fi + cat "$OUT"/$ARCH/MD5SUMS >> "$PUBDIRISO"/$ARCH/$DATE/MD5SUMS.tmp + mv "$PUBDIRISO"/$ARCH/$DATE/MD5SUMS.tmp "$PUBDIRISO"/$ARCH/$DATE/MD5SUMS + mv "$OUT"/$ARCH/$NETINSTNAME "$PUBDIRISO"/$ARCH/$DATE/ + rm -f "$PUBDIRISO"/$ARCH/current + ln -s $DATE "$PUBDIRISO"/$ARCH/current +# Remove the oldest daily builds, leave only the last 3 + NUM=`/bin/ls -d "$PUBDIRISO"/$ARCH/20*/|wc -l` + NUM=$(($NUM-3)) + /bin/ls -d "$PUBDIRISO"/$ARCH/20*/|head -n $NUM|tr '\012' '\000'|xargs -0 /bin/rm -rf -- + +else + +# NORMAL STUFF, IE NORMAL ISO and JIGDO + +# ISO STUFF + +if [ "$DOJIGDO" != 2 ] +then +# make the dirs we'll use to publish things + mkdir -p "$OUT"/isos/$ARCH +# rename the isos from .raw to .iso and put them in their place + for i in "$OUT"/*.raw + do + I=${i##*/} + mv $i "$OUT"/isos/$ARCH/${I%.raw}.iso + done +# change images names from raw to iso on the MD5SUMS + sed 's/raw$/iso/' "$OUT"/MD5SUMS >"$OUT"/isos/$ARCH/MD5SUMS +# Remove the old version and publish the new one + rm -rf "$PUBDIRISO"/isos/$ARCH/ + mkdir -p "$PUBDIRISO"/isos/ + mv "$OUT"/isos/$ARCH/ "$PUBDIRISO"/isos/$ARCH/ +fi + +# JIGDO STUFF + +if [ "$DOJIGDO" != 0 ] +then +# make the dirs we'll use to publish things + mkdir -p "$OUT"/jigdo-area/$ARCH/jigdotemplates +# change images names from raw to iso on the MD5SUMS + sed 's/raw$/iso/' "$OUT"/MD5SUMS >"$OUT"/jigdo-area/$ARCH/MD5SUMS +# put the templates aside + mv "$OUT"/*.template "$OUT"/jigdo-area/$ARCH/jigdotemplates +# finish up the jigdos (compress & move them) + gzip --best --no-name "$OUT"/*.jigdo + for i in "$OUT"/*.jigdo.gz;do mv $i ${i%.gz};done + mv "$OUT"/*.jigdo "$OUT"/jigdo-area/$ARCH/ +# Remove the old version and publish the new one + if [ "$IMAGETARGET" = "official_images" ] + then + rm -rf "$PUBDIRJIG"/jigdo-area/source/ + mkdir -p "$PUBDIRJIG"/jigdo-area/source/jigdotemplates/ + mv "$OUT"/jigdo-area/$ARCH/*-source-*.jigdo "$PUBDIRJIG"/jigdo-area/source/ + grep "source-.*.iso$" "$OUT"/jigdo-area/$ARCH/MD5SUMS > "$PUBDIRJIG"/jigdo-area/source/MD5SUMS + mv "$OUT"/jigdo-area/$ARCH/jigdotemplates/*-source-*.template "$PUBDIRJIG"/jigdo-area/source/jigdotemplates/ + grep -v "source-.*.iso$" "$OUT"/jigdo-area/$ARCH/MD5SUMS > "$OUT"/jigdo-area/$ARCH/MD5SUMS.new + mv "$OUT"/jigdo-area/$ARCH/MD5SUMS.new "$OUT"/jigdo-area/$ARCH/MD5SUMS + fi + rm -rf "$PUBDIRJIG"/jigdo-area/$ARCH/ + mkdir -p "$PUBDIRJIG"/jigdo-area/ + mv "$OUT"/jigdo-area/$ARCH/ "$PUBDIRJIG"/jigdo-area/$ARCH/ +# Calculate snapshot path, remove the old snapshot and publish the new one +# SNAP=${JIGDOFALLBACKURLS##Debian=http://us.cdimage.debian.org/cdimage/} +# SNAP=${PUBDIRJIG%%testing*}${SNAP%%/Debian*} +# mkdir -p "$SNAP" +# mv "$OUT"/snapshot/* "$SNAP" +# Remove the oldest snapshots, leave only the last 2 +# NUM=`/bin/ls -d "${SNAP%%/$DATE}"/20*/|wc -l` +# NUM=$(($NUM-2)) +# /bin/ls -d "${SNAP%%/$DATE}"/20*/|head -n $NUM|tr '\012' '\000'|xargs -0 /bin/rm -rf -- +fi + +fi + +# Mail the cd sizes of the cds +if [ -e "../.debian-cd.mail" ] +then + MYEMAIL=`grep -v "^#" "../.debian-cd.mail"` +else + MYEMAIL=`whoami` +fi + for i in $MYEMAIL + do + grep "extents written" "$LOG/$INSTALLER_CD$DI$ARCH"|mail -s "testingcds $INSTALLER_CD$DI$ARCH went ok" "$i" + done + overview "$INSTALLER_CD$DI$ARCH" success +else +if [ -e "../.debian-cd.mail" ] +then + MYEMAIL=`grep -v "^#" "../.debian-cd.mail"` +else + MYEMAIL=`whoami` +fi + for i in $MYEMAIL + do + echo Error when trying to create cds for $INSTALLER_CD$DI$ARCH|mail -s "testingcds $INSTALLER_CD$DI$ARCH has failed" "$i" + done + overview "$INSTALLER_CD$DI$ARCH" failed +fi