Backport more fixes from trunk:

* Size-override code updates
 * Make source DVDs fit better
 * Add more checksum types to the generated Release files
 * Tone down messages about missing include/exclude files

squeeze/build.sh

@@ -7,7 +7,8 @@ if [ -z "$CF" ] ; then
 fi
 . $CF
 
-echo "Using CONF from $CF"
+START=`date -u`
+echo "$START: Using CONF from $CF."
 
 if [ -z "$COMPLETE" ] ; then
     export COMPLETE=1
@@ -84,3 +85,6 @@ make $IMAGETARGET
 if [ "$IMAGESUMS"x = 1x ]; then
 	make imagesums
 fi
+
+END=`date -u`
+echo "$END: Finished."

squeeze/debian/changelog

@@ -1,4 +1,4 @@
-debian-cd (3.1.6) UNRELEASED; urgency=low
+debian-cd (3.1.6) UNRELEASED-backport; urgency=low
 
   * Add the missing accent in my family name in Uploaders.
   * Drop Frans Pop from Uploaders, RIP.
@@ -8,6 +8,20 @@ debian-cd (3.1.6) UNRELEASED; urgency=low
     x86. Closes: #612224
   * In the boot prompt for powerpc, make install64 more prominent.
     Closes: #614583
+  * Add support for over-riding disc size on demand, used for some of the
+    squeeze release builds.
+  * Add disk size definitions for 1/2/4/8 GB USB sticks.
+  * Extend the size-override code to allow to override an individual disc
+    in the set (e.g. to make the first DVD fit on a 4GB USB
+    stick). Closes: #612074
+  * Special-case builds of source-only discs: if we think we've
+    overflowed the disc size, try to carry on for a few more packages
+    to get closer to actually filling the disc. Closes: #613751.
+  * Add more checksum types to generated Release files; go from md5 only
+    to md5, sha1, sha256 and sha512.
+  * tools/start_new_disc: tone down messages about missing
+    {BASE,UDEB}_{INCLUDE,EXCLUDE} files from ERROR to WARNING.
+    Closes: #615043
 
  -- Raphaël Hertzog <hertzog@debian.org>  Fri, 04 Feb 2011 09:59:21 +0100
 

squeeze/tools/make_disc_trees.pl

@@ -6,6 +6,7 @@
 
 use strict;
 use Digest::MD5;
+use Digest::SHA;
 use File::stat;
 use File::Find;
 use File::Basename;
@@ -20,6 +21,7 @@ my $mkisofs_dirs = "";
 my (@arches, @arches_nosrc, @overflowlist, @pkgs_added);
 my (@exclude_packages, @unexclude_packages, @excluded_package_list);
 my %firmware_package;
+my $current_checksum_type = "";
 
 undef @pkgs_added;
 undef @exclude_packages;
@@ -115,6 +117,7 @@ my $size_check = "";
 # Constants used for space calculations
 my $MiB = 1048576;
 my $MB = 1000000;
+my $GB = 1000000000;
 my $blocksize = 2048;
 my ($maxdiskblocks, $diskdesc);
 my $cddir;
@@ -165,7 +168,7 @@ if ($archlist =~ /m68k/ || $archlist =~ /powerpc/) {
     print LOG "arches require HFS hybrid, multiplying sizes by $hfs_mult and marking $hfs_extra blocks for HFS use\n";
 }
 
-print "Starting to lay out packages into $disktype ($diskdesc) images: $maxdiskblocks 2K-blocks maximum per image\n";
+print "Starting to lay out packages into images:\n";
 
 if (-e "$bdir/firmware-packages") {
     open(FWLIST, "$bdir/firmware-packages") or die "Unable to read firmware-packages file!\n";
@@ -188,9 +191,8 @@ while (defined (my $pkg = <INLIST>)) {
             last;
         }
         print LOG "Starting new disc $disknum at " . `date` . "\n";
-
         start_disc();
-
+        print LOG "  Specified size: $diskdesc, $maxdiskblocks 2K-blocks maximum\n";
         print "  Placing packages into image $disknum\n";
         if ( -e "$bdir/$disknum.mkisofs_opts" ) {
             open(OPTS, "<$bdir/$disknum.mkisofs_opts");
@@ -292,12 +294,20 @@ while (defined (my $pkg = <INLIST>)) {
                 # Put this package first on the next disc
                 push (@overflowlist, $pkg);
             }
-            finish_disc($cddir, "");
+            # Special-case for source-only discs where we don't care
-
+            # about the ordering. If we're doing a source-only build
-            # And reset, to start the next disc
+            # and we've overflowed, allow us to carry on down the list
-            $size = 0;
+            # for a while to fill more space. Stop when we've skipped
-            $disknum++;
+            # 5 packages (arbitrary choice of number!) #613751
-            undef(@pkgs_added);
+            if (!($archlist eq "source") or (scalar @overflowlist >= 5)) {
+                finish_disc($cddir, "");
+                # And reset, to start the next disc
+                $size = 0;
+                $disknum++;
+                undef(@pkgs_added);
+            } else {
+                print LOG "SOURCE DISC: continuing on to see if anything else will fit, " . scalar @overflowlist . " packages on the overflow list at this point\n";
+            }
         } else {
             $pkgs_this_cd++;
             $pkgs_done++;
@@ -491,16 +501,23 @@ sub add_missing_Packages {
 	}
 }
 
-sub md5_file {
+sub checksum_file {
 	my $filename = shift;
-	my ($md5, $st);
+	my $alg = shift;
+	my ($checksum, $st);
 
-	open(MD5FILE, $filename) or die "Can't open '$filename': $!\n";
+	open(CHECKFILE, $filename) or die "Can't open '$filename': $!\n";
-	binmode(MD5FILE);
+	binmode(CHECKFILE);
-	$md5 = Digest::MD5->new->addfile(*MD5FILE)->hexdigest;
+	if ($alg eq "md5") {
-	close(MD5FILE);
+	    $checksum = Digest::MD5->new->addfile(*CHECKFILE)->hexdigest;
+	} elsif ($alg =~ /^sha\d+$/) {
+	    $checksum = Digest::SHA->new($alg)->addfile(*CHECKFILE)->hexdigest;
+	} else {
+	    die "checksum_file: unknown alorithm $alg!\n";
+	}
+	close(CHECKFILE);
 	$st = stat($filename) || die "Stat error on '$filename': $!\n";
-	return ($md5, $st->size);
+	return ($checksum, $st->size);
 }
 
 sub recompress {
@@ -515,24 +532,42 @@ sub recompress {
 	}
 }	
 
-sub md5_files_for_release {
+sub find_and_checksum_files_for_release {
-	my ($md5, $size, $filename);
+	my ($checksum, $size, $filename);
 
 	$filename = $File::Find::name;
 
 	if ($filename =~ m/\/.*\/(Packages|Sources|Release)/o) {
 		$filename =~ s/^\.\///g;
-		($md5, $size) = md5_file($_);
+		($checksum, $size) = checksum_file($_, $current_checksum_type);
-		printf RELEASE " %s %8d %s\n", $md5, $size, $filename;
+		printf RELEASE " %s %8d %s\n", $checksum, $size, $filename;
 	}
 }	
 
+sub checksum_files_for_release {
+    # ICK: no way to pass arguments to the
+    # find_and_checksum_files_for_release() function that I can see,
+    # so using a global here...
+	print RELEASE "MD5Sum:\n";
+	$current_checksum_type = "md5";
+	find (\&find_and_checksum_files_for_release, ".");
+	print RELEASE "SHA1:\n";
+	$current_checksum_type = "sha1";
+	find (\&find_and_checksum_files_for_release, ".");
+	print RELEASE "SHA256:\n";
+	$current_checksum_type = "sha256";
+	find (\&find_and_checksum_files_for_release, ".");
+	print RELEASE "SHA512:\n";
+	$current_checksum_type = "sha512";
+	find (\&find_and_checksum_files_for_release, ".");
+}    
+
 sub md5_files_for_md5sum {
 	my ($md5, $size, $filename);
 
 	$filename = $File::Find::name;
 	if (-f $_) {
-		($md5, $size) = md5_file($_);
+		($md5, $size) = checksum_file($_, "md5");
 		printf MD5LIST "%s  %s\n", $md5, $filename;
 	}
 }
@@ -541,6 +576,8 @@ sub get_disc_size {
     my $hook;
     my $error = 0;
     my $reserved = 0;
+    my $chosen_disk = $disktype;
+    my $disk_size_hack = "";
 
     if (defined($ENV{'RESERVED_BLOCKS_HOOK'})) {
         $hook = $ENV{'RESERVED_BLOCKS_HOOK'};
@@ -553,34 +590,62 @@ sub get_disc_size {
         print "  Reserving $reserved blocks on CD $disknum\n";
     }
 
+    # See if we've been asked to switch sizes for the whole set
+    $disk_size_hack = $ENV{'FORCE_CD_SIZE'} || "";
+    if ($disk_size_hack) {
+       print LOG "HACK HACK HACK: FORCE_CD_SIZE found:\n";
+       print LOG "  forcing use of a $disk_size_hack disk instead of $chosen_disk\n";
+       $chosen_disk = $disk_size_hack;
+    }
+
+    # If we're asked to do a specific size for *this* disknum, over-ride again
+    $disk_size_hack = $ENV{"FORCE_CD_SIZE$disknum"} || "";
+    if ($disk_size_hack) {
+       print LOG "HACK HACK HACK: FORCE_CD_SIZE$disknum found:\n";
+       print LOG "  forcing use of a $disk_size_hack disk instead of $chosen_disk\n";
+       $chosen_disk = $disk_size_hack;
+    }
+
     # Calculate the maximum number of 2K blocks in the output images
-    if ($disktype eq "BC") {
+    if ($chosen_disk eq "BC") {
         $maxdiskblocks = int(680 * $MB / $blocksize) - $reserved;
         $diskdesc = "businesscard";
-    } elsif ($disktype eq "NETINST") {
+    } elsif ($chosen_disk eq "NETINST") {
         $maxdiskblocks = int(680 * $MB / $blocksize) - $reserved;
         $diskdesc = "netinst";
-    } elsif ($disktype =~ /CD$/) {
+    } elsif ($chosen_disk =~ /CD$/) {
         $maxdiskblocks = int(680 * $MB / $blocksize) - $reserved;
         $diskdesc = "650MiB CD";
-    } elsif ($disktype eq "CD700") {
+    } elsif ($chosen_disk eq "CD700") {
         $maxdiskblocks = int(737 * $MB / $blocksize) - $reserved;
         $diskdesc = "700MiB CD";
-    } elsif ($disktype eq "DVD") {
+    } elsif ($chosen_disk eq "DVD") {
         $maxdiskblocks = int(4700 * $MB / $blocksize) - $reserved;
         $diskdesc = "4.7GB DVD";
-    } elsif ($disktype eq "DLDVD") {
+    } elsif ($chosen_disk eq "DLDVD") {
         $maxdiskblocks = int(8500 * $MB / $blocksize) - $reserved;
         $diskdesc = "8.5GB DVD";
-    } elsif ($disktype eq "BD") {
+    } elsif ($chosen_disk eq "BD") {
 		# Useable capacity, found by checking some disks
         $maxdiskblocks = 11230000 - $reserved;
         $diskdesc = "25GB BD";
-    } elsif ($disktype eq "DLBD") {
+    } elsif ($chosen_disk eq "DLBD") {
 		# Useable capacity, found by checking some disks
         $maxdiskblocks = 23652352 - $reserved;
         $diskdesc = "50GB DLBD";
-    } elsif ($disktype eq "CUSTOM") {
+    } elsif ($chosen_disk eq "STICK1GB") {
+        $maxdiskblocks = int(1 * $GB / $blocksize) - $reserved;
+        $diskdesc = "1GB STICK";
+    } elsif ($chosen_disk eq "STICK2GB") {
+        $maxdiskblocks = int(2 * $GB / $blocksize) - $reserved;
+        $diskdesc = "2GB STICK";
+    } elsif ($chosen_disk eq "STICK4GB") {
+        $maxdiskblocks = int(4 * $GB / $blocksize) - $reserved;
+        $diskdesc = "4GB STICK";
+    } elsif ($chosen_disk eq "STICK8GB") {
+        $maxdiskblocks = int(8 * $GB / $blocksize) - $reserved;
+        $diskdesc = "8GB STICK";
+    } elsif ($chosen_disk eq "CUSTOM") {
         $maxdiskblocks = $ENV{'CUSTOMSIZE'}  - $reserved || 
             die "Need to specify a custom size for the CUSTOM disktype\n";
         $diskdesc = "User-supplied size";
@@ -604,6 +669,8 @@ sub start_disc {
 
 	get_disc_size();
 
+    print "Starting new \"$archlist\" $disktype $disknum at $basedir/$codename/CD$disknum\n";
+    print "  Specified size for this image: $diskdesc, $maxdiskblocks 2K-blocks maximum\n";
 	# Grab all the early stuff, apart from dirs that will change later
 	print "  Starting the md5sum.txt file\n";
 	chdir $cddir;
@@ -699,9 +766,8 @@ sub finish_disc {
 	print "  Finishing off the Release file\n";
 	chdir "dists/$codename";
 	open(RELEASE, ">>Release") or die "Failed to open Release file: $!\n";
-	print RELEASE "MD5Sum:\n";
 	find (\&recompress, ".");
-	find (\&md5_files_for_release, ".");
+	checksum_files_for_release();
 	close(RELEASE);
 	chdir("../..");
 

squeeze/tools/start_new_disc

@@ -64,8 +64,6 @@ else
     export VOLID="$VOLID_BASE M-A $DISKNUM"
 fi
 
-echo "Starting new $ARCHDEF $DISKTYPE $DISKNUM at $CDDIR"
-
 mkdir -p $CDDIR
 echo "  Adding the required directories"
 
@@ -192,14 +190,14 @@ if [ $DISKNUM = 1 ] ; then
                 if [ -r "$UDEB_INCLUDE" ] ; then
                     cat "$UDEB_INCLUDE" >> "$CDDIR/.disk/udeb_include"
                 else
-                    echo "ERROR: Unable to read UDEB_INCLUDE file $UDEB_INCLUDE"
+                    echo "WARNING: Unable to read UDEB_INCLUDE file $UDEB_INCLUDE"
                 fi
             fi
             if [ -n "$UDEB_EXCLUDE" ] ; then
                 if [ -r "$UDEB_EXCLUDE" ] ; then
                     cat "$UDEB_EXCLUDE" >> "$CDDIR/.disk/udeb_exclude"
                 else
-                    echo "ERROR: Unable to read UDEB_EXCLUDE file $UDEB_EXCLUDE"
+                    echo "WARNING: Unable to read UDEB_EXCLUDE file $UDEB_EXCLUDE"
                 fi
             fi
 
@@ -208,14 +206,14 @@ if [ $DISKNUM = 1 ] ; then
                 if [ -r "$BASE_INCLUDE" ] ; then
                     cat "$BASE_INCLUDE" >> "$CDDIR/.disk/base_include"
                 else
-                    echo "ERROR: Unable to read BASE_INCLUDE file $BASE_INCLUDE"
+                    echo "WARNING: Unable to read BASE_INCLUDE file $BASE_INCLUDE"
                 fi
             fi
             if [ -n "$BASE_EXCLUDE" ] ; then
                 if [ -r "$BASE_EXCLUDE" ] ; then
                     cat "$BASE_EXCLUDE" >> "$CDDIR/.disk/base_exclude"
                 else
-                    echo "ERROR: Unable to read BASE_EXCLUDE file $BASE_EXCLUDE"
+                    echo "WARNING: Unable to read BASE_EXCLUDE file $BASE_EXCLUDE"
                 fi
             fi
         fi