Checking in the changes needed for the woodyr3 update CDs.

debian/changelog

@@ -3,6 +3,12 @@ debian-cd (2.2.21) UNRELEASED; urgency=low
   * Joey Hess
     - Exclude all the new amd64 kernel images from cd #1, no space.
     - Forgot to exclude old kernel-image-2.6.7-1-686-smp from cd #1.
+  * Steve McIntyre
+    - Substantially updated update-cd:
+		- Cope with Woody r3 updates taking more than 1 CD per
+	      architecture.
+        - (Optionally) use JTE mkisofs extensions to create jigdo
+		  files directly.
 
  -- Joey Hess <joeyh@debian.org>  Mon, 25 Oct 2004 03:03:36 -0400
 

update-cd

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 #
 # $Id$
 # 
@@ -8,7 +8,11 @@
 # Quick and dirty script to create update CDs for people to upgrade
 # from potato (2.2r0) to the latest release level
 #
+
+set -e
+
 # Configuration goes here.
+
 # Where is your mirror?
 MIRROR=${MIRROR:-/mirror/debian}
 
@@ -16,13 +20,17 @@ MIRROR=${MIRROR:-/mirror/debian}
 NONUS=${NONUS:-/mirror/debian-non-US}
 
 # Do you want non-free? 1 for yes, 0 for no
-NONFREE=0
+#NONFREE=1
 
 # What release version is this?
-VER=3.0r2
+VER=3.0r3
 
 # Path to use with mkisofs/mkhybrid
 MKISOFS=${MKISOFS:-"mkhybrid"}
+MKISOFS=mkisofs-jte
+
+# Use JTE extensions to mkisofs to make jigdo files?
+JTE=1
 
 # The working directory to use. MUST be on the same partition as the mirror.
 TDIR=/mirror/debian-cd
@@ -31,13 +39,14 @@ TDIR=/mirror/debian-cd
 CODENAME=woody
 
 # Path where the images will be written
-OUT=/mirror/woody-update/r2
+OUT=/mirror/woody-update/r3
 
 # Location of the diff file to use to determine the changes. If you leave
 # this blank, we'll try to determine the changes from the ChangeLog files,
 # which is probably less accurate. 
-DIFF=/mirror/lists/$CODENAME/r0-r2.diff #~/r2.diff
+DIFF=/mirror/lists/$CODENAME/r0-r3.diff #~/r3.diff
 
+CDSIZE=620 # megabytes, leaving space for metadata
 REL=Debian"$VER"
 CLOG=dists/$CODENAME/ChangeLog
 NUCLOG=dists/$CODENAME/non-US/ChangeLog
@@ -46,66 +55,20 @@ DATE=`date +%Y%m%d`
 BASEDIR=`pwd`
 SECTS="main non-free contrib"
 VERBOSE=2
+GRAB_MD5=$BASEDIR/tools/grab_md5
 export VERBOSE BASEDIR SECTS
 
 ARCHLIST="alpha arm hppa i386 ia64 m68k mips mipsel powerpc s390 sparc src" # all dealt with specially
 
 export TDIR NONFREE NONUS VER MIRROR CODENAME OUT BASEDIR
 
+CDSIZE=$(($CDSIZE * 1024))
+
 scanpackages=$BASEDIR/tools/scanpackages.old
 scansources=$BASEDIR/tools/scansources.old
 set_mkisofs_opts=$BASEDIR/tools/set_mkisofs_opts
 addfiles=$BASEDIR/tools/add_files
 
-adddirs () {
-    ROOTDIR=$1
-    shift
-    FILELIST=$1
-    for FILE in `cat $FILELIST`
-    do
-        DIR=`echo $FILE | sed 's?/[-_\.A-Za-z0-9\+]*$??'`
-        if [ ! -d $ROOTDIR/$DIR ] ; then
-            mkdir -p $ROOTDIR/$DIR
-        fi
-    done
-}
-
-copy_trees () {
-    CDDIR=$1
-    shift
-    SRCLIST=$1
-    shift
-    BINLIST=$1
-    shift
-    ARCHES=$*
-    echo "    Copying \"$ARCHES\" files into $UPD/$CDDIR"
-    for ARCH in $ARCHES
-    do
-        case "$ARCH" in
-            src)
-                NUM=`cat $SRCLIST | wc -l`
-                if [ $NUM -gt 0 ] ; then
-                    echo "      $ARCH into $UPD/$CDDIR"
-                    adddirs $UPD/$CDDIR $SRCLIST
-                    cat $SRCLIST | xargs $addfiles $UPD/$CDDIR $MIRROR
-                else
-                    echo "      No updates needed for $ARCH"
-                fi                
-                ;;
-            *)
-                NUM=`cat $BINLIST | grep _$ARCH.deb | wc -l`
-                if [ $NUM -gt 0 ] ; then
-                    echo "      $ARCH into $UPD/$CDDIR"
-                    adddirs $UPD/$CDDIR $BINLIST
-                    cat $BINLIST | grep _$ARCH.deb | xargs $addfiles $UPD/$CDDIR $MIRROR
-                else
-                    echo "      No updates needed for $ARCH"
-                fi                
-                ;;
-        esac
-    done
-}
-
 create_control_files () {
     CDDIR=$1
     shift
@@ -141,84 +104,150 @@ create_control_files () {
     done
 }
 
+copy_file () {
+    ROOTDIR=$1
+	shift
+	FILE=$1
+    DIR=`echo $FILE | sed 's?/[-_\.A-Za-z0-9\+]*$??'`
+    if [ ! -d $ROOTDIR/$DIR ] ; then
+        mkdir -p $ROOTDIR/$DIR
+    fi
+	if [ -e $MIRROR/$FILE ] ; then
+		cp -l $MIRROR/$FILE $ROOTDIR/$FILE
+	elif [ -e $NONUS/$FILE ] ; then
+		cp -l $NONUS/$FILE $ROOTDIR/$FILE
+	else
+		echo
+		echo "File $FILE not found!"
+		exit 1
+	fi
+}
+
+make_cd () {
+	CDNUM=$1
+	shift
+	THISNUM=$1
+	shift
+	THISARCH=$1
+	shift
+	SRCFILES=$1
+	shift
+	ARCHFILES=$1
+	shift
+	ALLFILES=$1
+	echo
+	echo "  Creating Packages and Sources files for CD$CDNUM"
+	cd $UPD
+	create_control_files CD$CDNUM $THISARCH
+	echo "  Creating image for CD$CDNUM ($SRCFILES source files, $ARCHFILES $ARCH debs, $ALLFILES all debs)"
+	echo -n "    "
+	if [ $JTE = 1 ] ; then
+		${MKISOFS} -J -r -V "Debian $VER update CD" -o \
+			/dev/null \
+			-jigdo-jigdo $OUT/$CODENAME-update-$THISARCH-$THISNUM.jigdo \
+			-jigdo-template $OUT/$CODENAME-update-$THISARCH-$THISNUM.template \
+			-jigdo-map Debian=$MIRROR/ \
+			-jigdo-map Non-US=$NONUS/ \
+			-jigdo-force-md5 /pool/ \
+			-md5-list $UPD/md5-check \
+			$UPD/CD$CDNUM 2>&1 | grep "extents written"
+	else
+		${MKISOFS} -J -r -V "Debian $VER update CD" -o \
+		$OUT/$CODENAME-update-$THISARCH-$THISNUM.raw \
+		$UPD/CD$CDNUM 2>&1 | grep "extents written"
+	fi
+}
+
+
 echo Cleaning up
 rm -rf $UPD
 mkdir $UPD
 CDNUM=1
 
 cd $MIRROR
-echo Creating main-section list
+echo Creating file list
 
-if [ -e $DIFF ] ; then
+if [ ! -e $DIFF ] ; then
-	grep -v non-US $DIFF >$UPD/list
+	echo "NO DIFF FOUND. ABORT!"
-	egrep -e \\.deb$ $UPD/list >$UPD/bin-list
+	exit 1
-	egrep -e \\.gz$ -e \\.dsc$ $UPD/list >$UPD/src-list
+fi
-else
+
-	egrep -e \\.deb$ $CLOG >$UPD/bin-list
+cp $DIFF $UPD/list
-	egrep -e \\.gz$ -e \\.dsc$ $CLOG >$UPD/src-list
+if [ "$NONUS"x == ""x ] ; then
+	echo "Removing non-US files from the list"
+	grep -v non-US $UPD/list > $UPD/list1
+	mv -f $UPD/list1 $UPD/list
 fi
 if [ "$NONFREE"x != "1"x ] ; then
-	echo Removing non-free
+	echo "Removing non-free files from the list"
-	grep -v non-free $UPD/bin-list > $UPD/bin-list1
+	grep -v non-free $UPD/list > $UPD/list1
-	mv -f $UPD/bin-list1 $UPD/bin-list
+	mv -f $UPD/list1 $UPD/list
-	grep -v non-free $UPD/src-list > $UPD/src-list1
-	mv -f $UPD/src-list1 $UPD/src-list
-fi
-
-if [ "$NONUS"x != ""x ] ; then
-	echo Creating non-US list
-	cd $NONUS
-	if [ -e $DIFF ] ; then
-		grep non-US $DIFF >$UPD/nu-list
-		egrep -e \\.deb$ $UPD/nu-list >$UPD/bin-nu-list
-		egrep -e \\.gz$ -e \\.dsc$ $UPD/nu-list >$UPD/src-nu-list
-	else
-		egrep -e \\.deb$ $NUCLOG | sed 's?^stable?dists/woody?g' > $UPD/bin-nu-list
-		egrep -e \\.gz$ -e \\.dsc$ $NUCLOG | sed 's?^stable?dists/woody?g' > $UPD/src-nu-list
-	fi
-
-	if [ "$NONFREE"x != "1"x ] ; then
-		echo Removing non-free
-		grep -v non-free $UPD/bin-nu-list > $UPD/bin-nu-list1
-		mv -f $UPD/bin-nu-list1 $UPD/bin-nu-list
-		grep -v non-free $UPD/src-nu-list > $UPD/src-nu-list1
-		mv -f $UPD/src-nu-list1 $UPD/src-nu-list
-	fi
-
 fi
 
 for THISARCH in $ARCHLIST
 do
-    if [ $THISARCH = "src" ] ; then
+	THISNUM=0
-        echo "Creating CD$CDNUM for source"
+
-    else
+	rm -f $UPD/md5-check
-        echo "Creating CD$CDNUM for binary-$THISARCH and binary-all"
+	$GRAB_MD5 $MIRROR $THISARCH $CODENAME $UPD/md5-check
-    fi
+	if [ "$NONUS"x != ""x ] ; then
-    mkdir $UPD/CD$CDNUM $UPD/CD$CDNUM/.disk
+		$GRAB_MD5 $NONUS $THISARCH $CODENAME $UPD/md5-check
-    INFO="Debian GNU/Linux $VER Update CD $DATE: $THISARCH"
+	fi
-    echo $INFO > $UPD/CD$CDNUM/.disk/info
+
-    echo "  Copying normal files"
+	for file in `cat $UPD/list`
-    if [ $THISARCH = "src" ] ; then
+	do
-        copy_trees CD$CDNUM $UPD/src-list $UPD/bin-list $THISARCH
+		if [ ! -d $UPD/CD$CDNUM ] ; then
-    else
+			echo
-        copy_trees CD$CDNUM $UPD/src-list $UPD/bin-list $THISARCH all
+			cd $MIRROR
-    fi
+			THISNUM=$((THISNUM + 1))
-    if [ "$NONUS"x != ""x ] ; then
+			if [ $THISARCH = "src" ] ; then
-        echo "  Copying non-US files"
+				echo "Creating CD$CDNUM for source (part $THISNUM)"
-        if [ $THISARCH = "src" ] ; then
+			else
-            copy_trees CD$CDNUM $UPD/src-nu-list $UPD/bin-nu-list $THISARCH
+				echo "Creating CD$CDNUM for binary-$THISARCH (part $THISNUM)"
-        else
+			fi
-            copy_trees CD$CDNUM $UPD/src-nu-list $UPD/bin-nu-list $THISARCH all
+			mkdir $UPD/CD$CDNUM $UPD/CD$CDNUM/.disk
-        fi
+			INFO="Debian GNU/Linux $VER Update CD $DATE: $THISARCH CD $THISNUM"
-    fi
+	   		echo $INFO > $UPD/CD$CDNUM/.disk/info
-    echo "  Creating Packages and Sources files"
+			SIZE_USED=0
-    cd $UPD
+			SRCFILES=0
-    create_control_files CD$CDNUM $THISARCH
+			ARCHFILES=0
-    echo "  Creating image for CD$CDNUM"
+			ALLFILES=0
-    echo -n "    "
+		fi
-    ${MKISOFS} -J -r -V "Debian $VER update CD" -o $OUT/$CODENAME-update-$THISARCH.raw $UPD/CD$CDNUM 2>&1 | grep "extents written"
+
-    CDNUM=$(($CDNUM + 1))
+		SIZE_USED=`du -sk $UPD/CD$CDNUM | awk '{print $1}'`
+
+		if [ $SIZE_USED -gt $CDSIZE ] ; then
+			make_cd $CDNUM $THISNUM $THISARCH $SRCFILES $ARCHFILES $ALLFILES
+			CDNUM=$(($CDNUM + 1))
+			continue
+		fi
+
+        case "$THISARCH" in
+            src)
+				case "$file" in
+					*.gz|*.dsc)
+						copy_file $UPD/CD$CDNUM $file
+						SRCFILES=$(($SRCFILES + 1))
+						;;
+				esac
+				echo -en "\r$SRCFILES copied"
+				;;
+			*)
+				case "$file" in
+					*_$THISARCH.deb)
+						copy_file $UPD/CD$CDNUM $file
+						ARCHFILES=$(($ARCHFILES + 1))
+						;;
+					*_all.deb)
+						copy_file $UPD/CD$CDNUM $file
+						ALLFILES=$(($ALLFILES + 1))
+						;;
+				esac
+				echo -en "\r$THISARCH:$ARCHFILES all:$ALLFILES copied"
+				;;
+		esac
+	done
+	make_cd $CDNUM $THISNUM $THISARCH $SRCFILES $ARCHFILES $ALLFILES
+	CDNUM=$(($CDNUM + 1))
 done
 
-
-