From 7462c548dcdc201de5ed0c93f336d17ccf58af58 Mon Sep 17 00:00:00 2001
From: cd-builder user <debian-cd@lists.debian.org>
Date: Wed, 10 Sep 2008 22:16:18 +0000
Subject: [PATCH]   * Improve control over production of iso and jigdo files.
 Replace the     simple DOJIGDO config option with MAXISOS and MAXJIGDOS,
 allowing     more flexibility. Allows us to only produce a small number of
 iso     images for less-popular architectures but still produce all the jigdo
     files, saving a lot of disk space.

---
 CONF.sh                  |  15 ++--
 Makefile                 |  10 +--
 build.sh                 |  13 +++-
 debian/CONF.sh           |  15 ++--
 debian/changelog         |   5 ++
 tools/imagesums          |  88 +++++++++++++----------
 tools/make_disc_trees.pl |  44 +++++++++++-
 tools/make_image         | 147 +++++++++++++++++++++------------------
 8 files changed, 205 insertions(+), 132 deletions(-)

diff --git a/CONF.sh b/CONF.sh
index 2b807cde..f81169bb 100644
--- a/CONF.sh
+++ b/CONF.sh
@@ -29,7 +29,6 @@ unset ISOLINUX          || true
 unset EXCLUDE           || true
 unset NORECOMMENDS      || true
 unset NOSUGGESTS        || true
-unset DOJIGDO           || true
 unset JIGDOTEMPLATEURL  || true
 unset JIGDOFALLBACKURLS || true
 unset JIGDOINCLUDEURLS  || true
@@ -199,11 +198,15 @@ export NORECOMMENDS=1
 # package on the CD.  The default is 'true'.
 #export NOSUGGESTS=1
 
-# Produce jigdo files:
-# 0/unset = Don't do jigdo at all, produce only the full iso image.
-# 1 = Produce both the iso image and jigdo stuff.
-# 2 = Produce only the jigdo stuff
-export DOJIGDO=1
+# Produce iso/jigdo files: specify how many iso/jigdo files should be
+# produced in your set, or "ALL". Replaces the old "DOJIGDO" setting
+# with something much more flexible
+if [ "$MAXISOS"x = ""x ] ; then
+    export MAXISOS="ALL"
+fi
+if [ "$MAXJIGDOS"x = ""x ] ; then
+    export MAXJIGDOS="ALL"
+fi
 
 # HTTP/FTP URL for directory where you intend to make the templates
 # available. You should not need to change this; the default value ""
diff --git a/Makefile b/Makefile
index 435c2c33..0a160c3b 100755
--- a/Makefile
+++ b/Makefile
@@ -32,9 +32,6 @@ endif
 ifndef HOOK
 HOOK=$(BASEDIR)/tools/$(CODENAME).hook
 endif
-ifndef DOJIGDO
-export DOJIGDO=0
-endif
 
 export BUILD_DATE=$(shell date -u +%Y%m%d-%H:%M)
 export ARCHES_NOSRC=$(shell echo $(ARCHES) | sed 's/source//')
@@ -362,13 +359,8 @@ image-trees: ok genlist
 	fi
 	$(Q)$(BASEDIR)/tools/make_disc_trees.pl $(BASEDIR) $(MIRROR) $(TDIR) $(CODENAME) "$(ARCHES)" $(MKISOFS)
 
-# DOJIGDO actions   (for both binaries and source)
-#    0    isofile
-#    1    isofile + jigdo, cleanup_jigdo
-#    2    jigdo, cleanup_jigdo
-#
 images: ok $(OUT) $(BDIR)/md5-check
-	$(Q)$(make_image) "$(BDIR)" "$(ARCHES)" "$(OUT)" "$(DOJIGDO)" "$(DEBVERSION)" "$(MIRROR)" "$(MKISOFS)" "$(MKISOFS_OPTS)" "$(JIGDO_OPTS)" "$(jigdo_cleanup)"
+	$(Q)$(make_image) "$(BDIR)" "$(ARCHES)" "$(OUT)" "$(DEBVERSION)" "$(MIRROR)" "$(MKISOFS)" "$(MKISOFS_OPTS)" "$(JIGDO_OPTS)" "$(jigdo_cleanup)"
 
 check-number-given:
 	@test -n "$(CD)" || (echo "Give me a CD=<num> parameter !" && false)
diff --git a/build.sh b/build.sh
index 25fee7dd..3fcb6172 100755
--- a/build.sh
+++ b/build.sh
@@ -1,4 +1,4 @@
-#!/bin/sh -e
+#!/bin/bash -e
 
 # Script to build images for one or more architectures and/or source
 
@@ -55,6 +55,17 @@ if [ -z "$IMAGETARGET" ] ; then
     IMAGETARGET="official_images"
 fi
 echo " ... building the images; using target(s) \"$IMAGETARGET\""
+
+NUMISOS="up to $MAXISOS"
+if [ "$MAXISOS"x = "all"x ] || [ "$MAXISOS"x = "ALL"x ] ; then
+    NUMISOS="all available"
+fi
+NUMJIGDOS="up to $MAXJIGDOS"
+if [ "$MAXJIGDOS"x = "all"x ] || [ "$MAXJIGDOS"x = "ALL"x ] ; then
+    NUMJIGDOS="all available"
+fi
+echo "Building $NUMJIGDOS jigdos and $NUMISOS isos for $ARCH $DISKTYPE"
+
 make $IMAGETARGET
 
 make imagesums
diff --git a/debian/CONF.sh b/debian/CONF.sh
index 2b807cde..f81169bb 100644
--- a/debian/CONF.sh
+++ b/debian/CONF.sh
@@ -29,7 +29,6 @@ unset ISOLINUX          || true
 unset EXCLUDE           || true
 unset NORECOMMENDS      || true
 unset NOSUGGESTS        || true
-unset DOJIGDO           || true
 unset JIGDOTEMPLATEURL  || true
 unset JIGDOFALLBACKURLS || true
 unset JIGDOINCLUDEURLS  || true
@@ -199,11 +198,15 @@ export NORECOMMENDS=1
 # package on the CD.  The default is 'true'.
 #export NOSUGGESTS=1
 
-# Produce jigdo files:
-# 0/unset = Don't do jigdo at all, produce only the full iso image.
-# 1 = Produce both the iso image and jigdo stuff.
-# 2 = Produce only the jigdo stuff
-export DOJIGDO=1
+# Produce iso/jigdo files: specify how many iso/jigdo files should be
+# produced in your set, or "ALL". Replaces the old "DOJIGDO" setting
+# with something much more flexible
+if [ "$MAXISOS"x = ""x ] ; then
+    export MAXISOS="ALL"
+fi
+if [ "$MAXJIGDOS"x = ""x ] ; then
+    export MAXJIGDOS="ALL"
+fi
 
 # HTTP/FTP URL for directory where you intend to make the templates
 # available. You should not need to change this; the default value ""
diff --git a/debian/changelog b/debian/changelog
index a909a36b..b3feb7f2 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -95,6 +95,11 @@ debian-cd (3.0.5) UNRELEASED; urgency=low
     first 64 characters, so normal Joliet sorting fails.
   * Cope better with source/binary package mismatches in
     tools/merge_package_lists.
+  * Improve control over production of iso and jigdo files. Replace the
+    simple DOJIGDO config option with MAXISOS and MAXJIGDOS, allowing
+    more flexibility. Allows us to only produce a small number of iso
+    images for less-popular architectures but still produce all the jigdo
+    files, saving a lot of disk space.
 
  -- Frans Pop <fjp@debian.org>  Mon, 04 Aug 2008 18:59:31 +0200
 
diff --git a/tools/imagesums b/tools/imagesums
index 5fd59703..398cdfe1 100755
--- a/tools/imagesums
+++ b/tools/imagesums
@@ -5,44 +5,56 @@ cd $1
 :> MD5SUMS
 :> SHA1SUMS
 
-case $DOJIGDO in
-	0)
-		for file in `find * -name \*.iso`
-		do
-			md5sum $file >> MD5SUMS
-			sha1sum $file >> SHA1SUMS
-		done
-		;;
-	1|2)
-		for file in `find * -name \*.jigdo`
-		do
+# 2 calls to find here to get the ordering right - we *really* want to
+# get the jigdo files first
+for file in `find * -name \*.jigdo` `find * -name \*.iso`; do
+    case $file in
+        *.jigdo)
             iso=${file%%.jigdo}.iso
+            JIGDO=1
+            ;;
+        *.iso)
+            iso=$file
+            JIGDO=0
+            ;;
+        *)
+            echo "Found unexpected file $file!"
+            exit 1
+            ;;
+    esac
 
-			grep -q $iso MD5SUMS
-            if [ $? -ne 0 ] ; then
-			    MD5=`zcat -f $file | awk '/Image Hex MD5Sum/ {print $5}'`
-                if [ "$MD5"x != ""x ] ; then
-                    echo "$MD5  $iso" >> MD5SUMS
-                else
-                    echo "Jigdo file does not contain the Image MD5, calculating by hand"
-                    md5sum $iso >> MD5SUMS
-                fi
-            fi
+    grep -q $iso MD5SUMS
+    if [ $? -ne 0 ] ; then
+        MD5=""
+        if [ "$JIGDO" == 1 ] ; then
+            MD5=`zcat -f $file | awk '/Image Hex MD5Sum/ {print $5}'`
+        fi
+        if [ "$MD5"x != ""x ] ; then
+            echo "$MD5  $iso" >> MD5SUMS
+            echo "Jigdo says:"
+            echo "$MD5  $iso"
+        else
+            echo "No Jigdo help for md5, doing it the long way"
+            md5sum $iso >> MD5SUMS
+            grep $iso MD5SUMS
+        fi
+    fi
 
-			grep -q $iso SHA1SUMS
-            if [ $? -ne 0 ] ; then
-			    SHA1=`zcat -f $file | awk '/Image Hex SHA1Sum/ {print $5}'`
-                if [ "$SHA1"x != ""x ] ; then
-                    echo "$SHA1  $iso" >> SHA1SUMS
-                else
-                    echo "Jigdo file does not contain the Image SHA1, calculating by hand"
-                    sha1sum $iso >> SHA1SUMS
-                fi
-            fi
-		done
-		;;
-	*)
-		echo "DOJIGDO not defined!"
-		exit 1
-		;;
-esac
+    grep -q $iso SHA1SUMS
+    if [ $? -ne 0 ] ; then
+        SHA1=""
+        if [ "$JIGDO" == 1 ] ; then
+            SHA1=`zcat -f $file | awk '/Image Hex SHA1Sum/ {print $5}'`
+        fi
+        if [ "$SHA1"x != ""x ] ; then
+            echo "$SHA1  $iso" >> SHA1SUMS
+            echo "Jigdo says:"
+            echo "$SHA1  $iso"
+        else
+            echo "No Jigdo help for sha1, doing it the long way"
+            sha1sum $iso >> SHA1SUMS
+            grep $iso SHA1SUMS
+        fi
+    fi
+
+done
diff --git a/tools/make_disc_trees.pl b/tools/make_disc_trees.pl
index a641819b..9f7dc10b 100755
--- a/tools/make_disc_trees.pl
+++ b/tools/make_disc_trees.pl
@@ -11,7 +11,8 @@ use File::Find;
 use Compress::Zlib;
 
 my %pkginfo;
-my ($basedir, $mirror, $tdir, $codename, $archlist, $mkisofs, $maxcds, $extranonfree);
+my ($basedir, $mirror, $tdir, $codename, $archlist, $mkisofs, $maxcds,
+    $maxisos, $maxjigdos, $extranonfree);
 my $mkisofs_opts = "";
 my $mkisofs_dirs = "";
 my (@arches, @arches_nosrc, @overflowlist, @pkgs_added);
@@ -40,10 +41,47 @@ my $link_copy = $ENV{'COPYLINK'} || 0;
 
 require "$basedir/tools/link.pl";
 
+$maxcds = 9999;
+
+# MAXCDS is the hard limit on the MAXIMUM number of images to
+# make. MAXJIGDOS and MAXISOS can only make this number smaller; we
+# will use the higher of those 2 numbers as the last image to go to,
+# if they're set
+
 if (defined($ENV{'MAXCDS'})) {
 	$maxcds = $ENV{'MAXCDS'};
 } else {
-	$maxcds = 0;
+	$maxcds = 9999;
+}
+
+if (defined($ENV{'MAXISOS'})) {
+	$maxisos = $ENV{'MAXISOS'};
+    if ($maxisos =~ 'ALL' || $maxisos =~ 'all') {
+        $maxisos = 9999;
+    }
+} else {
+	$maxisos = 9999;
+}
+
+if (defined($ENV{'MAXJIGDOS'})) {
+	$maxjigdos = $ENV{'MAXJIGDOS'};
+    if ($maxjigdos =~ 'ALL' || $maxjigdos =~ 'all') {
+        $maxjigdos = 9999;
+    }
+} else {
+	$maxjigdos = 9999;
+}
+
+if ($maxisos > $maxjigdos) {
+    $maxjigdos = $maxisos;
+}
+
+if ($maxjigdos > $maxisos) {
+    $maxisos = $maxjigdos;
+}
+
+if ($maxisos < $maxcds) {
+    $maxcds = $maxisos;
 }
 
 if (defined($ENV{'EXTRANONFREE'})) {
@@ -132,7 +170,7 @@ while (defined (my $pkg = <INLIST>)) {
 	$cddir = "$bdir/CD$disknum";
 	my $opt;
 	if (! -d $cddir) {
-		if (($maxcds > 0 ) && ($disknum > $maxcds)) {
+		if ($disknum > $maxcds) {
 			print LOG "Disk $disknum is beyond the configured MAXCDS of $maxcds; exiting now...\n";
 			$max_done = 1;
 			last;
diff --git a/tools/make_image b/tools/make_image
index c191a809..5e3661b2 100755
--- a/tools/make_image
+++ b/tools/make_image
@@ -9,13 +9,11 @@ set -e
 DIR=$1
 ARCHES="$2"
 OUT=$3
-DOJIGDO=$4
-DEBVERSION=$5
-MIRROR=$6
-MKISOFS=$7
-MKISOFS_OPTS=$8
-JIGDO_OPTS=$9
-shift
+DEBVERSION=$4
+MIRROR=$5
+MKISOFS=$6
+MKISOFS_OPTS=$7
+JIGDO_OPTS=$8
 JIGDO_CLEANUP=$9
 
 NUM_ARCHES=`echo $ARCHES | wc -w`
@@ -49,7 +47,6 @@ for file in $FILES
 do
 	dir=${file%%.volid}
 	n=${dir##$DIR/}
-	num=$n
 	dir=$DIR/CD$n
 
     cd $dir
@@ -72,74 +69,86 @@ do
 
 	date
 
+    # Work out whether we want ISO, jigdo or both
+    if [ "$MAXISOS" = "all" ] || [ "$MAXISOS" = "ALL" ] || [ "$n" -le "$MAXISOS" ] ; then
+        ISO_WANTED=1
+    else
+        ISO_WANTED=0
+    fi
+    if [ "$MAXJIGDOS" = "all" ] || [ "$MAXJIGDOS" = "ALL" ] || [ "$n" -le "$MAXJIGDOS" ] ; then
+        JIGDO_WANTED=1
+    else
+        JIGDO_WANTED=0
+    fi
+
 	# Actually make the ISO/jigdo images. Long command lines
 	# here... :-(
-	case $DOJIGDO in
-	
-		0) # No jigdo files, just straight ISO
-			echo $MKISOFS $MKISOFS_OPTS -V "$volid" -o $OUT/$OUTFILE.iso $opts $dirs CD$n
-			echo $MKISOFS $MKISOFS_OPTS -V "$volid" -o $OUT/$OUTFILE.iso $opts $dirs CD$n > CD$n/.disk/mkisofs
-			$MKISOFS $MKISOFS_OPTS -V "$volid" -o $OUT/$OUTFILE.iso $opts $dirs CD$n
-			;;
+    if [ $JIGDO_WANTED = 0 ] && [ $ISO_WANTED = 1 ] ; then
 
+        echo $MKISOFS $MKISOFS_OPTS -V "$volid" -o $OUT/$OUTFILE.iso $opts $dirs CD$n
+        echo $MKISOFS $MKISOFS_OPTS -V "$volid" -o $OUT/$OUTFILE.iso $opts $dirs CD$n > CD$n/.disk/mkisofs
+        $MKISOFS $MKISOFS_OPTS -V "$volid" -o $OUT/$OUTFILE.iso $opts $dirs CD$n
 
-		1) # jigdo files _and_ ISO
-			echo $MKISOFS $MKISOFS_OPTS -V "$volid" \
-				-o $OUT/$OUTFILE.iso \
-				-jigdo-jigdo $OUT/$OUTFILE.jigdo \
-				-jigdo-template $OUT/$OUTFILE.template \
-				-jigdo-map Debian=$MIRROR/ \
-				-jigdo-exclude boot$n \
-				-md5-list $DIR/md5-check \
-				$JIGDO_OPTS $opts $dirs CD$n
-			echo $MKISOFS $MKISOFS_OPTS -V "$volid" \
-				-o $OUT/$OUTFILE.iso \
-				-jigdo-jigdo $OUT/$OUTFILE.jigdo \
-				-jigdo-template $OUT/$OUTFILE.template \
-				-jigdo-map Debian=$MIRROR/ \
-				-jigdo-exclude boot$n \
-				-md5-list $DIR/md5-check \
-				$JIGDO_OPTS $opts $dirs CD$n > CD$n/.disk/mkisofs
-			$MKISOFS $MKISOFS_OPTS -V "$volid" \
-				-o $OUT/$OUTFILE.iso \
-				-jigdo-jigdo $OUT/$OUTFILE.jigdo \
-				-jigdo-template $OUT/$OUTFILE.template \
-				-jigdo-map Debian=$MIRROR/ \
-				-jigdo-exclude boot$n \
-				-md5-list $DIR/md5-check \
-				$JIGDO_OPTS $opts $dirs CD$n
-			;;
+    elif [ $JIGDO_WANTED = 1 ] && [ $ISO_WANTED = 1 ] ; then
 
-		2) # jigdo only, no ISO
-			echo $MKISOFS $MKISOFS_OPTS -V "$volid" \
-				-o /dev/null \
-				-jigdo-jigdo $OUT/$OUTFILE.jigdo \
-				-jigdo-template $OUT/$OUTFILE.template \
-				-jigdo-map Debian=$MIRROR/ \
-				-jigdo-exclude boot$n \
-				-md5-list $DIR/md5-check \
-				$JIGDO_OPTS $opts $dirs CD$n
-			echo $MKISOFS $MKISOFS_OPTS -V "$volid" \
-				-o /dev/null \
-				-jigdo-jigdo $OUT/$OUTFILE.jigdo \
-				-jigdo-template $OUT/$OUTFILE.template \
-				-jigdo-map Debian=$MIRROR/ \
-				-jigdo-exclude boot$n \
-				-md5-list $DIR/md5-check \
-				$JIGDO_OPTS $opts $dirs CD$n > CD$n/.disk/mkisofs
-			$MKISOFS $MKISOFS_OPTS -V "$volid" \
-				-o /dev/null \
-				-jigdo-jigdo $OUT/$OUTFILE.jigdo \
-				-jigdo-template $OUT/$OUTFILE.template \
-				-jigdo-map Debian=$MIRROR/ \
-				-jigdo-exclude boot$n \
-				-md5-list $DIR/md5-check \
-				$JIGDO_OPTS $opts $dirs CD$n
-			;;
-	esac
+        echo $MKISOFS $MKISOFS_OPTS -V "$volid" \
+            -o $OUT/$OUTFILE.iso \
+            -jigdo-jigdo $OUT/$OUTFILE.jigdo \
+            -jigdo-template $OUT/$OUTFILE.template \
+            -jigdo-map Debian=$MIRROR/ \
+            -jigdo-exclude boot$n \
+            -md5-list $DIR/md5-check \
+            $JIGDO_OPTS $opts $dirs CD$n
+        echo $MKISOFS $MKISOFS_OPTS -V "$volid" \
+            -o $OUT/$OUTFILE.iso \
+            -jigdo-jigdo $OUT/$OUTFILE.jigdo \
+            -jigdo-template $OUT/$OUTFILE.template \
+            -jigdo-map Debian=$MIRROR/ \
+            -jigdo-exclude boot$n \
+            -md5-list $DIR/md5-check \
+            $JIGDO_OPTS $opts $dirs CD$n > CD$n/.disk/mkisofs
+        $MKISOFS $MKISOFS_OPTS -V "$volid" \
+            -o $OUT/$OUTFILE.iso \
+            -jigdo-jigdo $OUT/$OUTFILE.jigdo \
+            -jigdo-template $OUT/$OUTFILE.template \
+            -jigdo-map Debian=$MIRROR/ \
+            -jigdo-exclude boot$n \
+            -md5-list $DIR/md5-check \
+            $JIGDO_OPTS $opts $dirs CD$n
+
+    elif [ $JIGDO_WANTED = 1 ] && [ $ISO_WANTED = 0 ] ; then
+
+        echo $MKISOFS $MKISOFS_OPTS -V "$volid" \
+            -o /dev/null \
+            -jigdo-jigdo $OUT/$OUTFILE.jigdo \
+            -jigdo-template $OUT/$OUTFILE.template \
+            -jigdo-map Debian=$MIRROR/ \
+            -jigdo-exclude boot$n \
+            -md5-list $DIR/md5-check \
+            $JIGDO_OPTS $opts $dirs CD$n
+        echo $MKISOFS $MKISOFS_OPTS -V "$volid" \
+            -o /dev/null \
+            -jigdo-jigdo $OUT/$OUTFILE.jigdo \
+            -jigdo-template $OUT/$OUTFILE.template \
+            -jigdo-map Debian=$MIRROR/ \
+            -jigdo-exclude boot$n \
+            -md5-list $DIR/md5-check \
+            $JIGDO_OPTS $opts $dirs CD$n > CD$n/.disk/mkisofs
+        $MKISOFS $MKISOFS_OPTS -V "$volid" \
+            -o /dev/null \
+            -jigdo-jigdo $OUT/$OUTFILE.jigdo \
+            -jigdo-template $OUT/$OUTFILE.template \
+            -jigdo-map Debian=$MIRROR/ \
+            -jigdo-exclude boot$n \
+            -md5-list $DIR/md5-check \
+            $JIGDO_OPTS $opts $dirs CD$n
+
+    else
+        echo "Neither jigdo nor iso wanted for CD$n"
+    fi
 
 	# If we've made jigdo files, tweak them with extra info now
-	if [ "$DOJIGDO" != "0" ] ; then
+	if [ "$JIGDO_WANTED" = 1 ] ; then
 		$JIGDO_CLEANUP $OUT/$OUTFILE.jigdo \
 			$OUTFILE.iso $DIR/CD$n \
 			"`echo "$JIGDOTEMPLATEURL" | sed -e 's|%ARCH%|$ARCH|g'`$OUTFILE.template" \