Update the update-cd script for squeeze.

Started off as minor changes, but then also updated how things work in core
logic. We've been making broken over-sized update CDs for a while, as the
code didn't check for being over-full properly. Used to work better when we
had smaller packages, but now it's not working. Change to the method used in
the main debian-cd code: add packages one at a time until we go over, then
remove the last package. Works fine now.
This commit is contained in:
cd-builder user 2011-03-19 21:52:29 +00:00
parent 2f38ab7430
commit b1bdfb8a18
1 changed files with 176 additions and 141 deletions

View File

@ -24,27 +24,26 @@ MIRROR_AMD64=/org/cdbuilder.debian.org/src/ftp/debian-amd64/debian
NONFREE=0 NONFREE=0
# What release version is this? # What release version is this?
VER=4.0r1 VER=6.0.1
# Is this an official CD? # Is this an official CD?
OFFICIAL=Official OFFICIAL=Official
# OFFICIAL="Unofficial snapshot" # OFFICIAL="Unofficial snapshot"
# Path to use with mkisofs/mkhybrid # Path to use with mkisofs/mkhybrid
#MKISOFS=${MKISOFS:-"mkhybrid"} MKISOFS="$PWD/../genisoimage -jigdo-template-compress bzip2 -r -checksum_algorithm_iso md5,sha1,sha256,sha512"
MKISOFS=$PWD/../mkisofs/usr/bin/mkisofs
# Use JTE extensions to mkisofs to make jigdo files? # Use JTE extensions to mkisofs to make jigdo files?
JTE=1 JTE=1
# The working directory to use. MUST be on the same partition as the mirror. # The working directory to use. MUST be on the same partition as the mirror.
TDIR=/org/cdbuilder.debian.org/src/deb-cd/tmp/etch-update/cd-work TDIR=/org/cdbuilder.debian.org/src/deb-cd/tmp/squeeze-update/cd-work
# Building etch cd set ... # Building squeeze cd set ...
CODENAME=etch CODENAME=squeeze
# Path where the images will be written # Path where the images will be written
OUT=/org/cdbuilder.debian.org/src/deb-cd/tmp/etch-update/cd-out OUT=/org/cdbuilder.debian.org/src/deb-cd/tmp/squeeze-update/cd-out
# Location(s) for the snapshot(s) # Location(s) for the snapshot(s)
SNAPSHOT='Debian=http://us.cdimage.debian.org/cdimage/snapshot/Debian/ --try-last' SNAPSHOT='Debian=http://us.cdimage.debian.org/cdimage/snapshot/Debian/ --try-last'
@ -57,23 +56,23 @@ else
fi fi
case $TYPE in case $TYPE in
cd|CD) cd|CD)
TYPE=cd TYPE=cd
TYPEUP=CD;; TYPEUP=CD;;
dvd|DVD) dvd|DVD)
TYPE=dvd TYPE=dvd
TYPEUP=DVD;; TYPEUP=DVD;;
*) *)
echo "Specify a disk type..." echo "Specify a disk type..."
exit 1;; exit 1;;
esac esac
# Location of the diff file to use to determine the changes. If you leave # Location of the diff file to use to determine the changes. If you leave
# this blank, we'll try to determine the changes from the ChangeLog files, # this blank, we'll try to determine the changes from the ChangeLog files,
# which is probably less accurate. # which is probably less accurate.
DIFF=/home/deb-cd/r0-r1.diff DIFF=/home/debian-cd/lists/squeeze/r0-r1.diff
if [ "$TYPE" = "cd" ] ; then if [ "$TYPE" = "cd" ] ; then
CDSIZE=610 # megabytes, leaving space for metadata CDSIZE=640 # megabytes, leaving space for metadata
else else
CDSIZE=4300 # megabytes, leaving space for metadata CDSIZE=4300 # megabytes, leaving space for metadata
fi fi
@ -89,14 +88,13 @@ VERBOSE=2
GRAB_MD5=$BASEDIR/tools/grab_md5 GRAB_MD5=$BASEDIR/tools/grab_md5
export VERBOSE BASEDIR SECTS export VERBOSE BASEDIR SECTS
FIRSTVER=`echo $VER | sed 's?r.*$?r0?g'` FIRSTVER=`echo $VER | cut -c 1-4`0
VERSPACE=`echo $VER | sed 's?r? r?g'`
CAPCODE=`perl -e "print ucfirst("$CODENAME")"` CAPCODE=`perl -e "print ucfirst("$CODENAME")"`
export FIRSTVER VERSPACE CAPCODE export FIRSTVER CAPCODE
if [ "$ARCHLIST"x = ""x ] ; then if [ "$ARCHLIST"x = ""x ] ; then
ARCHLIST="alpha amd64 arm hppa i386 ia64 mips mipsel powerpc s390 sparc source" # amd64 # all dealt with specially ARCHLIST="amd64 i386 ia64 mips mipsel powerpc s390 sparc source kfreebsd-amd64 kfreebsd-i386" # amd64 # all dealt with specially
fi fi
export TDIR NONFREE NONUS VER MIRROR CODENAME OUT BASEDIR export TDIR NONFREE NONUS VER MIRROR CODENAME OUT BASEDIR
@ -145,72 +143,118 @@ create_control_files () {
copy_file () { copy_file () {
ROOTDIR=$1 ROOTDIR=$1
shift shift
FILE=$1 FILE=$1
DIR=`dirname $FILE` DIR=`dirname $FILE`
if [ ! -d $ROOTDIR/$DIR ] ; then if [ ! -d $ROOTDIR/$DIR ] ; then
mkdir -p $ROOTDIR/$DIR mkdir -p $ROOTDIR/$DIR
fi fi
if [ -e $MIRROR/$FILE ] ; then if [ -e $MIRROR/$FILE ] ; then
cp -l $MIRROR/$FILE $ROOTDIR/$FILE cp -l $MIRROR/$FILE $ROOTDIR/$FILE
elif [ -e $NONUS/$FILE ] ; then elif [ -e $NONUS/$FILE ] ; then
cp -l $NONUS/$FILE $ROOTDIR/$FILE cp -l $NONUS/$FILE $ROOTDIR/$FILE
else else
echo echo
echo "File $FILE not found!" echo "File $FILE not found!"
exit 1 exit 1
fi fi
} }
make_cd () { make_cd () {
CDNUM=$1 CDNUM=$1
shift shift
THISNUM=$1 THISNUM=$1
shift shift
THISARCH=$1 THISARCH=$1
shift shift
SRCFILES=$1 SRCFILES=$1
shift shift
ARCHFILES=$1 ARCHFILES=$1
shift shift
ALLFILES=$1 ALLFILES=$1
echo echo
echo " Creating Packages and Sources files for CD$CDNUM" echo " Creating Packages and Sources files for CD$CDNUM"
cd $UPD cd $UPD
create_control_files CD$CDNUM $THISARCH create_control_files CD$CDNUM $THISARCH
echo " Creating image for CD$CDNUM ($SRCFILES source files, $ARCHFILES $ARCH debs, $ALLFILES all debs)" echo " Creating image for CD$CDNUM ($SRCFILES source files, $ARCHFILES $ARCH debs, $ALLFILES all debs)"
echo -n " " echo -n " "
if [ ! -d $OUT/$THISARCH/iso-$TYPE ] ; then if [ ! -d $OUT/$THISARCH/iso-$TYPE ] ; then
mkdir -p $OUT/$THISARCH/iso-$TYPE mkdir -p $OUT/$THISARCH/iso-$TYPE
fi
if [ $JTE = 1 ] ; then
if [ ! -d $OUT/$THISARCH/jigdo-$TYPE ] ; then
mkdir -p $OUT/$THISARCH/jigdo-$TYPE
fi fi
if [ $JTE = 1 ] ; then ${MKISOFS} -J -r -V "Debian $VER update $TYPEUP" -o \
if [ ! -d $OUT/$THISARCH/jigdo-$TYPE ] ; then $OUT/$THISARCH/iso-$TYPE/debian-update-$VER-$THISARCH-$TYPEUP-$THISNUM.iso \
mkdir -p $OUT/$THISARCH/jigdo-$TYPE -jigdo-jigdo $OUT/$THISARCH/jigdo-$TYPE/debian-update-$VER-$THISARCH-$TYPEUP-$THISNUM.jigdo \
fi -jigdo-template $OUT/$THISARCH/jigdo-$TYPE/debian-update-$VER-$THISARCH-$TYPEUP-$THISNUM.template \
${MKISOFS} -J -r -V "Debian $VER update $TYPEUP" -o \ -jigdo-map Debian=$MIRROR/ \
$OUT/$THISARCH/iso-$TYPE/debian-update-$VER-$THISARCH-$TYPEUP-$THISNUM.iso \ -jigdo-map Non-US=$NONUS/ \
-jigdo-jigdo $OUT/$THISARCH/jigdo-$TYPE/debian-update-$VER-$THISARCH-$TYPEUP-$THISNUM.jigdo \ -jigdo-force-md5 /pool/ \
-jigdo-template $OUT/$THISARCH/jigdo-$TYPE/debian-update-$VER-$THISARCH-$TYPEUP-$THISNUM.template \ -md5-list $UPD/md5-check \
-jigdo-map Debian=$MIRROR/ \ $UPD/CD$CDNUM 2>&1 | grep "extents written"
-jigdo-map Non-US=$NONUS/ \ $BASEDIR/tools/jigdo_cleanup \
-jigdo-force-md5 /pool/ \ $OUT/$THISARCH/jigdo-$TYPE/debian-update-$VER-$THISARCH-$TYPEUP-$THISNUM.jigdo \
-md5-list $UPD/md5-check \ debian-update-$VER-$THISARCH-$TYPEUP-$THISNUM.iso \
$UPD/CD$CDNUM 2>&1 | grep "extents written" $OUT debian-update-$VER-$THISARCH-$TYPEUP-$THISNUM.template \
$BASEDIR/tools/jigdo_cleanup \ "Debian GNU/Linux $VER - $OFFICIAL $THISARCH $FIRSTVER->$VER update $TYPEUP #$THISNUM"
$OUT/$THISARCH/jigdo-$TYPE/debian-update-$VER-$THISARCH-$TYPEUP-$THISNUM.jigdo \ echo $SNAPSHOT >> $OUT/$THISARCH/jigdo-$TYPE/debian-update-$VER-$THISARCH-$TYPEUP-$THISNUM.jigdo
debian-update-$VER-$THISARCH-$TYPEUP-$THISNUM.iso \ # Make sure that the ISO is as new/newer than the jigdo file; #587774
$OUT debian-update-$VER-$THISARCH-$TYPEUP-$THISNUM.template \
"Debian GNU/Linux $VERSPACE - $OFFICIAL $THISARCH $FIRSTVER->$VER update $TYPEUP #$THISNUM"
echo $SNAPSHOT >> $OUT/$THISARCH/jigdo-$TYPE/debian-update-$VER-$THISARCH-$TYPEUP-$THISNUM.jigdo
# Make sure that the ISO is as new/newer than the jigdo file; #587774
touch $OUT/$THISARCH/iso-$TYPE/debian-update-$VER-$THISARCH-$TYPEUP-$THISNUM.iso touch $OUT/$THISARCH/iso-$TYPE/debian-update-$VER-$THISARCH-$TYPEUP-$THISNUM.iso
else
${MKISOFS} -J -r -V "Debian $VER update $TYPEUP" -o \ MD5=`cat $OUT/$THISARCH/jigdo-$TYPE/debian-update-$VER-$THISARCH-$TYPEUP-$THISNUM.jigdo | \
$OUT/iso-$TYPE/$CODENAME-update-$THISARCH-$TYPEUP-$THISNUM.iso \ awk '/Image Hex MD5Sum/ {print $5}'`
$UPD/CD$CDNUM 2>&1 | grep "extents written" echo "$MD5 debian-update-$VER-$THISARCH-$TYPEUP-$THISNUM.iso" >> $OUT/$THISARCH/jigdo-$TYPE/MD5SUMS.update
fi echo "$MD5 debian-update-$VER-$THISARCH-$TYPEUP-$THISNUM.iso" >> $OUT/$THISARCH/iso-$TYPE/MD5SUMS.update
for SHA_SIZE in 1 256 512; do
SHA=`cat $OUT/$THISARCH/jigdo-$TYPE/debian-update-$VER-$THISARCH-$TYPEUP-$THISNUM.jigdo | \
awk "/Image Hex SHA${SHA_SIZE}Sum/ {print \\$5}"`
if [ "$SHA"x = ""x ] ; then
echo "SHA fail"
exit 1
fi
echo "$SHA debian-update-$VER-$THISARCH-$TYPEUP-$THISNUM.iso" >> $OUT/$THISARCH/jigdo-$TYPE/SHA${SHA_SIZE}SUMS.update
echo "$SHA debian-update-$VER-$THISARCH-$TYPEUP-$THISNUM.iso" >> $OUT/$THISARCH/iso-$TYPE/SHA${SHA_SIZE}SUMS.update
done
gzip -9 $OUT/$THISARCH/jigdo-$TYPE/debian-update-$VER-$THISARCH-$TYPEUP-$THISNUM.jigdo
mv -v $OUT/$THISARCH/jigdo-$TYPE/debian-update-$VER-$THISARCH-$TYPEUP-$THISNUM.jigdo.gz \
$OUT/$THISARCH/jigdo-$TYPE/debian-update-$VER-$THISARCH-$TYPEUP-$THISNUM.jigdo
else
${MKISOFS} -J -r -V "Debian $VER update $TYPEUP" -o \
$OUT/iso-$TYPE/$CODENAME-update-$THISARCH-$TYPEUP-$THISNUM.iso \
$UPD/CD$CDNUM 2>&1 | grep "extents written"
fi
} }
add_file () {
file=$1
case "$THISARCH" in
source)
case "$file" in
*.gz|*.dsc)
copy_file $UPD/CD$CDNUM $file
SRCFILES=$(($SRCFILES + 1))
;;
esac
echo -en "\r$SRCFILES copied"
;;
*)
case "$file" in
*_$THISARCH.deb)
copy_file $UPD/CD$CDNUM $file
ARCHFILES=$(($ARCHFILES + 1))
;;
*_all.deb)
copy_file $UPD/CD$CDNUM $file
ALLFILES=$(($ALLFILES + 1))
;;
esac
echo -en "\r$THISARCH:$ARCHFILES all:$ALLFILES copied"
;;
esac
}
echo Cleaning up echo Cleaning up
rm -rf $UPD rm -rf $UPD
@ -220,88 +264,79 @@ CDNUM=1
echo Creating file list echo Creating file list
if [ ! -e $DIFF ] ; then if [ ! -e $DIFF ] ; then
echo "NO DIFF FOUND. ABORT!" echo "NO DIFF FOUND. ABORT!"
exit 1 exit 1
fi fi
cp $DIFF $UPD/list cp $DIFF $UPD/list
if [ "$NONUS"x == ""x ] ; then if [ "$NONUS"x == ""x ] ; then
echo "Removing non-US files from the list" echo "Removing non-US files from the list"
grep -v non-US $UPD/list > $UPD/list1 grep -v non-US $UPD/list > $UPD/list1
mv -f $UPD/list1 $UPD/list mv -f $UPD/list1 $UPD/list
fi fi
if [ "$NONFREE"x != "1"x ] ; then if [ "$NONFREE"x != "1"x ] ; then
echo "Removing non-free files from the list" echo "Removing non-free files from the list"
grep -v non-free $UPD/list > $UPD/list1 grep -v non-free $UPD/list > $UPD/list1
mv -f $UPD/list1 $UPD/list mv -f $UPD/list1 $UPD/list
fi
$GRAB_MD5 $MIRROR_NORM "$ARCHLIST" $CODENAME $CODENAME $UPD/md5-check
if [ "$NONUS"x != ""x ] ; then
$GRAB_MD5 $NONUS "$ARCHLIST" $CODENAME $CODENAME $UPD/md5-check
fi fi
for THISARCH in $ARCHLIST for THISARCH in $ARCHLIST
do do
THISNUM=0 THISNUM=0
MIRROR=$MIRROR_NORM MIRROR=$MIRROR_NORM
rm -f $UPD/md5-check rm -rf $OUT/$THISARCH/iso-$TYPE $OUT/$THISARCH/jigdo-$TYPE
$GRAB_MD5 $MIRROR $THISARCH $CODENAME $CODENAME $UPD/md5-check
if [ "$NONUS"x != ""x ] ; then for file in `cat $UPD/list`
$GRAB_MD5 $NONUS $THISARCH $CODENAME $CODENAME $UPD/md5-check do
if [ ! -d $UPD/CD$CDNUM ] ; then
echo
cd $MIRROR
THISNUM=$((THISNUM + 1))
if [ $THISARCH = "source" ] ; then
echo "Creating $TYPEUP$CDNUM for source (part $THISNUM)"
else
echo "Creating $TYPEUP$CDNUM for binary-$THISARCH (part $THISNUM)"
fi
mkdir $UPD/CD$CDNUM $UPD/CD$CDNUM/.disk
INFO="Debian GNU/Linux $VER Update $TYPEUP $DATE: $THISARCH $TYPEUP $THISNUM"
echo $INFO > $UPD/CD$CDNUM/.disk/info
SIZE_USED=0
SRCFILES=0
ARCHFILES=0
ALLFILES=0
fi
if [ "$BACKOUT_FILE"x != ""x ] ; then
echo "Starting with backed-out file $BACKOUT_FILE"
add_file $BACKOUT_FILE
BACKOUT_FILE=""
fi fi
for file in `cat $UPD/list` add_file $file
do
if [ ! -d $UPD/CD$CDNUM ] ; then
echo
cd $MIRROR
THISNUM=$((THISNUM + 1))
if [ $THISARCH = "source" ] ; then
echo "Creating $TYPEUP$CDNUM for source (part $THISNUM)"
else
echo "Creating $TYPEUP$CDNUM for binary-$THISARCH (part $THISNUM)"
fi
mkdir $UPD/CD$CDNUM $UPD/CD$CDNUM/.disk
INFO="Debian GNU/Linux $VER Update $TYPEUP $DATE: $THISARCH $TYPEUP $THISNUM"
echo $INFO > $UPD/CD$CDNUM/.disk/info
SIZE_USED=0
SRCFILES=0
ARCHFILES=0
ALLFILES=0
fi
SIZE_USED=`du -sk $UPD/CD$CDNUM | awk '{print $1}'` SIZE_USED=`du -sk $UPD/CD$CDNUM | awk '{print $1}'`
if [ $SIZE_USED -gt $CDSIZE ] ; then
# Back out the last file added
echo
echo " Overfull: back out $file"
rm -f $UPD/CD$CDNUM/$file
DIR=`dirname $file`
rmdir $DIR > /dev/null 2>&1 || true
BACKOUT_FILE=$file
make_cd $CDNUM $THISNUM $THISARCH $SRCFILES $ARCHFILES $ALLFILES
CDNUM=$(($CDNUM + 1))
continue
fi
done
if [ $SIZE_USED -gt $CDSIZE ] ; then make_cd $CDNUM $THISNUM $THISARCH $SRCFILES $ARCHFILES $ALLFILES
make_cd $CDNUM $THISNUM $THISARCH $SRCFILES $ARCHFILES $ALLFILES CDNUM=$(($CDNUM + 1))
CDNUM=$(($CDNUM + 1))
continue
fi
case "$THISARCH" in
source)
case "$file" in
*.gz|*.dsc)
copy_file $UPD/CD$CDNUM $file
SRCFILES=$(($SRCFILES + 1))
;;
esac
echo -en "\r$SRCFILES copied"
;;
*)
case "$file" in
*_$THISARCH.deb)
copy_file $UPD/CD$CDNUM $file
ARCHFILES=$(($ARCHFILES + 1))
;;
*_all.deb)
copy_file $UPD/CD$CDNUM $file
ALLFILES=$(($ALLFILES + 1))
;;
esac
echo -en "\r$THISARCH:$ARCHFILES all:$ALLFILES copied"
;;
esac
done
make_cd $CDNUM $THISNUM $THISARCH $SRCFILES $ARCHFILES $ALLFILES
CDNUM=$(($CDNUM + 1))
done done