Add initial support for jigdo v2 format

Add the option to switch to format v2 which uses sha256 everywhere
instead of md5. We can't actually turn this on yet, need to wait for
users to upgrade a bit and get new v2-aware jigdo clients.

But this works as-is for generating v1 until we hit that switch - see
the new JIGOO_CHECKSUM option in CONF.sh

CONF.sh

@@ -181,6 +181,12 @@ export CONTRIB=1
 # partition than your source files.
 # export COPYLINK=1
 
+# Choose the checksum algorithm used in jigdo and template
+# files. Older jigdo tools can only support md5; but we want to move
+# to sha256 as a better checksum. Depends on xorriso 1.5.3+ to support
+# sha256
+export JIGDO_CHECKSUM="md5"
+
 # Options
 # export MKISOFS=mkisofs
 # export MKISOFS_OPTS="-r"		#For normal users

Makefile

@@ -78,7 +78,7 @@ check_backports_packages=$(BASEDIR)/tools/check_backports_packages
 sort_deps=$(BASEDIR)/tools/sort_deps
 md5sum=md5sum
 jigdo_cleanup=$(BASEDIR)/tools/jigdo_cleanup
-grab_md5=$(BASEDIR)/tools/grab_md5
+grab_checksums=$(BASEDIR)/tools/grab_checksums
 make_image=$(BASEDIR)/tools/make_image
 merge_package_lists=$(BASEDIR)/tools/merge_package_lists
 update_popcon=$(BASEDIR)/tools/update_popcon
@@ -208,7 +208,7 @@ dir-clean:
 	$(Q)rm -rf $(BDIR)/CD[1234567890]*
 	$(Q)rm -rf $(TASKDIR)
 	$(Q)rm -f $(BDIR)/*.filelist*
-	$(Q)rm -f  $(BDIR)/packages-stamp $(BDIR)/upgrade-stamp $(BDIR)/md5-check
+	$(Q)rm -f  $(BDIR)/packages-stamp $(BDIR)/upgrade-stamp $(BDIR)/checksum-check
 
 # Completely cleans the current arch tree
 realclean: distclean
@@ -488,7 +488,7 @@ image-trees: ok genlist
 	fi
 	$(Q)$(BASEDIR)/tools/make_disc_trees.pl $(BASEDIR) $(MIRROR) $(TDIR) $(CODENAME) "$(ARCHES)" "$(MKISOFS)" "$(MKISOFS_OPTS) $(JIGDO_OPTS)"
 
-images: ok $(OUT) $(BDIR)/md5-check
+images: ok $(OUT) $(BDIR)/checksum-check
 	$(Q)$(make_image) "$(BDIR)" "$(ARCHES)" "$(OUT)" "$(DEBVERSION)" "$(MIRROR)" "$(MKISOFS)" "$(MKISOFS_OPTS)" "$(JIGDO_OPTS)" "$(jigdo_cleanup)"
 
 check-number-given:
@@ -503,14 +503,14 @@ imagesums:
 
 ## MISC TARGETS ##
 
-$(BDIR)/md5-check: mirrorcheck	
+$(BDIR)/checksum-check: mirrorcheck
 
 mirrorcheck: ok
-	$(Q)$(grab_md5) $(MIRROR) "$(ARCHES)" $(CODENAME) $(DI_CODENAME) $(BDIR)/md5-check
+	$(Q)$(grab_checksums) ${JIGDO_CHECKSUM} $(MIRROR) "$(ARCHES)" $(CODENAME) $(DI_CODENAME) $(BDIR)/checksum-check
 	$(Q)for ARCH in $(ARCHES); do \
 		if [ -e $(BASEDIR)/data/$(CODENAME)/$$ARCH/extra-sources ]; then \
-			echo "Extra dedicated source added; need to grab source MD5 info too"; \
+			echo "Extra dedicated source added; need to grab source checksum info too"; \
-			$(grab_md5) $(MIRROR) source $(CODENAME) $(DI_CODENAME) $(BDIR)/md5-check; \
+			$(grab_checksums) $(JIGDO_CHECKSUM) $(MIRROR) source $(CODENAME) $(DI_CODENAME) $(BDIR)/checksum-check; \
 		fi; \
 	done
 

debian/changelog

@@ -4,6 +4,13 @@ debian-cd (3.1.28) UNRELEASED; urgency=medium
   * Add a "with firmware" text to the DISKINFO string when appropriate
     Closes: #944207
 
+  * Major changes to jigdo support - add support for the new
+    sha256-based v2 format so we can think about phasing out
+    dependencies on md5. Will still build using md5 stuff for
+    compatibility for now - we need a new xorriso release (1.5.3?) to
+    hit the archive and (more importantly!) users to get current jigdo
+    clients too before we can switch.
+
   [ Samuel Thibault ]
   * Drop support for xorriso versions before 1.2.2
   * Add hurd-i386 EFI support.

tools/grab_checksums

@@ -1,7 +1,7 @@
 #!/bin/sh
 #
-# grab_md5
+# grab_checksum
-# (c) 2004-2016 Steve McIntyre <steve@einval.com>
+# (c) 2004-2019 Steve McIntyre <steve@einval.com>
 #
 # GPL v2
 #
@@ -10,11 +10,25 @@
 
 set -e
 
-MIRROR=$1
+CHECKSUM=$1
-ARCHES="$2"
+MIRROR=$2
-CODENAME=$3
+ARCHES="$3"
-DI_CODENAME=$4
+CODENAME=$4
-OUT=$5
+DI_CODENAME=$5
+OUT=$6
+
+case $CHECKSUM in
+    md5|sha256)
+	# OK
+        ;;
+    *)
+	echo "$0: Unsupported checksum specified: $CHECKSUM"
+	echo "Abort"
+	exit 1
+	;;
+esac
+
+export CHECKSUM
 
 for ARCH in $ARCHES
 do
@@ -34,26 +48,32 @@ do
     case $ARCH in
         source)
             FILES=`find $LOCATIONS -name Sources.gz -o -name Sources.xz`
-            echo "Using MD5 sums from Sources files:"
+            echo "Using $CHECKSUM sums from Sources files:"
             echo $FILES
-            $BASEDIR/tools/catz $FILES | MIRROR=$MIRROR perl -ne '
+            $BASEDIR/tools/catz $FILES | \
+		MIRROR=$MIRROR CHECKSUM=$CHECKSUM perl -e '
                 chomp;
                 my %files;
                 my $dir;
-                my $mirror = $ENV{"MIRROR"};
                 my $filename;
+                my $mirror = $ENV{"MIRROR"};
+		my $checksum = $ENV{"CHECKSUM"};
                 while (<>) {
                     if (m/^ ([[:xdigit:]]{32}) (\d+) (\S+)/sg) {
                         $files{$3}{"md5"} = $1;
                         $files{$3}{"size"} = $2;
                     }
+                    if (m/^ ([[:xdigit:]]{64}) (\d+) (\S+)/sg) {
+                        $files{$3}{"sha256"} = $1;
+                        $files{$3}{"size"} = $2;
+                    }
                     if (m/^Directory: (\S+)/sg) {
                         $dir = $1;
                     }
                     if (m/^$/) {
                         for $filename (keys %files) {
                             printf("%s  %12s  %s/%s/%s\n",
-                                $files{$filename}{"md5"},
+                                $files{$filename}{$checksum},
                                 $files{$filename}{"size"},
                                 $mirror, $dir, $filename);
                         }
@@ -64,19 +84,21 @@ do
         *)
             FILES=`find $LOCATIONS -name Packages.gz \
                    -o -name Packages.xz | grep binary-$ARCH`
-            echo "Using MD5 sums from Packages files:"
+            echo "Using $CHECKSUM sums from Packages files:"
             echo $FILES
             NUM_FILES=`echo $FILES| wc -w`
             if [ $NUM_FILES -eq 1 ] ; then
                 echo "No files found for arch $ARCH. Abort!"
                 exit 1
             fi
-            $BASEDIR/tools/catz $FILES | MIRROR=$MIRROR perl -ne '
+            $BASEDIR/tools/catz $FILES | \
+		MIRROR=$MIRROR CHECKSUM=$CHECKSUM perl -e '
                 chomp;
                 my $mirror = $ENV{"MIRROR"};
+		my $checksum = $ENV{"CHECKSUM"};
                 my $filename;
                 my $size;
-                my $md5;
+                my $sum;
                 while (<>) {
                     if (m/^Filename: (\S+)/sg) {
                         $filename = $1;
@@ -84,11 +106,14 @@ do
                     if (m/^Size: (\S+)/sg) {
                         $size = $1;
                     }
-                    if (m/^MD5sum: (\S+)/sg) {
+		    if ($checksum eq "md5" and (m/^MD5sum: (\S+)/sg)) {
-                        $md5 = $1;
+                        $sum = $1;
+                    } elsif ($checksum eq "sha256" and (m/^SHA256: (\S+)/sg)) {
+                        $sum = $1;
                     }
                     if (m/^$/) {
-                        printf("%s  %12s  %s/%s\n", $md5, $size, $mirror, $filename);
+                        printf("%s  %12s  %s/%s\n", $sum, $size, $mirror, $filename);
+			$sum = "";
                     }
                 }' | sort | uniq >> $OUT
             # Use the new D-I images. Do NOT use the "current"
@@ -96,15 +121,19 @@ do
             for VER in $MIRROR/dists/$DI_CODENAME/main/installer-$ARCH/*
             do
                 if [ -d $VER ] && [ ! -L $VER ] ; then
+		    if [ $CHECKSUM = md5 ]; then
 			FILE=$VER/images/MD5SUMS
-                    echo "Using MD5 sums from d-i: $FILE"
+		    else
+			FILE=$VER/images/SHA256SUMS
+		    fi
+                    echo "Using $CHECKSUM sums from d-i: $FILE"
                     LOC=dists/$DI_CODENAME/main/installer-$ARCH/`basename $VER`/images
                     for ENTRY in `cat $FILE | sed 's/  /:/g'`
                     do
                         PATH=`echo $ENTRY | /bin/sed "s?^.*:\./?$MIRROR/$LOC/?g"`
-                        MD5=`echo $ENTRY | /bin/sed 's/:.*$//g'`
+                        CSUM=`echo $ENTRY | /bin/sed 's/:.*$//g'`
                         SIZE=`/usr/bin/stat -c %s $PATH`
-                        printf '%s  %12.12s  %s\n' $MD5 $SIZE $PATH
+                        printf '%s  %12.12s  %s\n' $CSUM $SIZE $PATH
                     done | sort | uniq >> $OUT
                  fi
             done
@@ -113,3 +142,4 @@ do
 done
 
 exit 0
+

tools/make_disc_trees.pl

@@ -203,9 +203,8 @@ while (defined (my $pkg = <INLIST>)) {
             $mkisofs_opts = "";
         }
         if ($disknum <= $maxjigdos) {
-            $mkisofs_opts = "$mkisofs_opts -jigdo-jigdo /dev/null";
+	    # Set things to /dev/null - we're only doing a
-            $mkisofs_opts = "$mkisofs_opts -jigdo-template /dev/null";
+	    # sizing run here
-            $mkisofs_opts = "$mkisofs_opts -md5-list /dev/null";
             $mkisofs_opts = "$mkisofs_opts -o /dev/null";
         }
         if ( -e "$bdir/$disknum.mkisofs_dirs" ) {

tools/make_image

@@ -42,6 +42,15 @@ last_minute_update () {
     done
 }
 
+# Grab the xorriso version and turn it into a number we can use
+xorriso_version() {
+    $MKISOFS --version 2>&1 | awk '
+	/^xorriso version/ {
+	    split($4, ver, ".")
+	    print ver[1]*10000+ver[2]*100+ver[3]
+	}'
+}
+
 for file in $FILES
 do
     dir=${file%%.volid}
@@ -82,8 +91,36 @@ do
         JIGDO_WANTED=0
     fi
 
+    # Work out common jigdo options if needed
+    if [ $JIGDO_WANTED = 1 ]; then
+	XOR_VER=$(xorriso_version)
+	if [ $XOR_VER -lt 10502 ]; then
+	    # Only supports jigdo format v1
+	    case "$JIGDO_CHECKSUM"x in
+		"md5"x|x)
+		    JIGDO_CHECKSUM_OPTS="-md5-list $DIR/checksum-check \
+		    			 -jigdo-force-md5 /pool/"
+		    ;;
+		*)
+		    echo "Older xorriso version $XOR_VER does not support jigdo checksum algorithm $JIGDO_CHECKSUM"
+		    exit 1
+		    ;;
+	    esac
+	else
+	    JIGDO_CHECKSUM_OPTS="-checksum-list $DIR/checksum-check \
+	                         -jigdo-checksum-algorithm $JIGDO_CHECKSUM \
+				 -jigdo-force-checksum /pool/"
+	fi
+
+	JIGDO_OPTS="$JIGDO_CHECKSUM_OPTS $JIGDO_OPTS \
+            -jigdo-jigdo $OUT/$OUTFILE.jigdo \
+            -jigdo-template $OUT/$OUTFILE.template \
+            -jigdo-map Debian=$MIRROR/ \
+            -jigdo-exclude boot$n"
+    fi
+
     # Actually make the ISO/jigdo images. Long command lines
-    # here... :-(
+    # here, with lots of stuff included :-(
     CMD=
     if [ $JIGDO_WANTED = 0 ] && [ $ISO_WANTED = 1 ] ; then
 
@@ -94,22 +131,12 @@ do
 
         CMD="$MKISOFS $MKISOFS_OPTS -V '$volid' \
             -o $OUT/$OUTFILE.iso \
-            -jigdo-jigdo $OUT/$OUTFILE.jigdo \
-            -jigdo-template $OUT/$OUTFILE.template \
-            -jigdo-map Debian=$MIRROR/ \
-            -jigdo-exclude boot$n \
-            -md5-list $DIR/md5-check \
             $JIGDO_OPTS $opts $dirs CD$n"
 
     elif [ $JIGDO_WANTED = 1 ] && [ $ISO_WANTED = 0 ] ; then
 
         CMD="$MKISOFS $MKISOFS_OPTS -V '$volid' \
             -o /dev/null \
-            -jigdo-jigdo $OUT/$OUTFILE.jigdo \
-            -jigdo-template $OUT/$OUTFILE.template \
-            -jigdo-map Debian=$MIRROR/ \
-            -jigdo-exclude boot$n \
-            -md5-list $DIR/md5-check \
             $JIGDO_OPTS $opts $dirs CD$n"
 
     else

update-cd

@@ -45,6 +45,8 @@ OUT=/srv/cdbuilder.debian.org/src/deb-cd/tmp/stretch-update/cd-out
 # Location(s) for the snapshot(s)
 SNAPSHOT='Debian=https://us.cdimage.debian.org/cdimage/snapshot/Debian/ --try-last'
 
+JIGOO_CHECKSUM=md5
+
 # Type of disc image, cd or dvd (lower case)
 if [ "$1"x = ""x ] ; then
     TYPE=cd
@@ -81,7 +83,7 @@ DATE=`date +%Y%m%d`
 BASEDIR=`pwd`
 SECTS="main non-free contrib"
 VERBOSE=2
-GRAB_MD5=$BASEDIR/tools/grab_md5
+GRAB_CHECKSUMS=$BASEDIR/tools/grab_checksums
 export VERBOSE BASEDIR SECTS
 
 FIRSTVER=`echo $VER | cut -c 1-4`0
@@ -164,7 +166,7 @@ make_cd () {
             -jigdo-template ${JIGDODIR}/${BASENAME}.template \
             -jigdo-map Debian=$MIRROR/ \
             -jigdo-force-md5 /pool/ \
-            -md5-list $UPD/md5-check \
+            -checksum-list $UPD/checksum-check \
             $UPD/CD$CDNUM 2>&1 | grep "extents written"
 #            $UPD/CD$CDNUM
 	if [ $? -ne 0 ] ; then
@@ -280,7 +282,7 @@ if [ "$NONFREE"x != "1"x ] ; then
     mv -f $UPD/list1 $UPD/list
 fi
 
-$GRAB_MD5 $MIRROR_NORM "$ARCHLIST all" $CODENAME $CODENAME $UPD/md5-check
+$GRAB_CHECKSUMS $JIGOO_CHECKSUM $MIRROR_NORM "$ARCHLIST all" $CODENAME $CODENAME $UPD/checksum-check
 
 for THISARCH in $ARCHLIST
 do