Extend the code that copes with too-large binary packages to cope with

too-large source packages too.

debian/changelog

@@ -47,9 +47,9 @@ debian-cd (3.1.3) UNRELEASED; urgency=low
     Mark Post and Bastian Blank for providing the pointers to the missing bits.
 
   [ Steve McIntyre ]
-  * Add support for coping with packages that are too large to fit. If we
+  * Add support for coping with binary and source packages that are too
-    have to remove packages and their dependencies, list them in
+    large to fit. If we have to remove packages and their dependencies,
-    README.excluded in the root of disc 1.
+    list them in README.excluded in the root of disc 1.
 
  -- Frans Pop <fjp@debian.org>  Sun, 23 Aug 2009 08:59:25 +0200
 

tools/grab_source_list

@@ -6,7 +6,7 @@ IN="$3"
 OUT="$4"
 
 echo "DONEDONE" > $BDIR/DONE
-awk '
+awk -v max_size=$MAX_PKG_SIZE '
 	/^Package:/ {
 				  srcname=$2
 				  srcs_done++
@@ -32,19 +32,39 @@ awk '
 				  next
 				}
 
+    /^Files:/ {
+                  in_files = 1
+                  next
+                }
+    /^ / {
+                  if (in_files) {
+                      size[srcname]+=$2
+                      next
+                  }
+                }
+
 	/.*/        {
+                    in_files = 0
 				    if (parsed) {
-						printf("source:%s:%s\n", component[$0], $0)
+                        if (size[$0] > max_size) {
+				           printf("source:%s:%s-SRCTOOBIG\n", component[$0], $0)
+                        } else {
+				           printf("source:%s:%s\n", component[$0], $0)
+                        }
 					}
 				}
 ' $ADIR/$CODENAME-source/apt-state/lists/*Sources $BDIR/DONE $IN > $BDIR/list.mid
 
 if [ "$NONFREE"x = "1"x ] ; then
-	cat $BDIR/list.mid > $OUT
+    grep -v SRCTOOBIG $BDIR/list.mid > $OUT
 else
-	grep -v :non-free: $BDIR/list.mid > $OUT
+    grep -v -e :non-free: -e SRCTOOBIG $BDIR/list.mid > $OUT
 fi
 
 if [ "$EXTRANONFREE"x = "1"x ] ; then 
-	grep :non-free: $BDIR/list.mid >> $OUT
+	grep :non-free: $BDIR/list.mid | grep -v SRCTOOBIG >> $OUT
 fi
+
+awk -F : '
+    /SRCTOOBIG/   { print $3 }
+' $BDIR/list.mid > $BDIR/sourcetoobig

tools/make_disc_trees.pl

@@ -213,7 +213,7 @@ while (defined (my $pkg = <INLIST>)) {
         # lists intersect and we should re-include some packages
         if (scalar @unexclude_packages && scalar @excluded_package_list) {
             foreach my $reinclude_pkg (@excluded_package_list) {
-                my ($arch, $component, $pkgname) = split /:/, $reinclude_pkg;
+                my ($arch, $component, $pkgname, $pkgsize) = split /:/, $reinclude_pkg;
                 foreach my $entry (@unexclude_packages) {
                     if (($pkgname =~ /^\Q$entry\E$/m)) {
                         print LOG "Re-including $reinclude_pkg due to match on \"\^$entry\$\"\n";
@@ -316,7 +316,7 @@ sub load_packages_cache {
 
     while (defined (my $pkg = <INLIST>)) {
         chomp $pkg;
-        my ($junk, $component, $pkgname) = split /:/, $pkg;
+        my ($junk, $component, $pkgname, $pkgsize) = split /:/, $pkg;
         push @pkglist, $pkgname;
     }
     close INLIST;
@@ -347,7 +347,7 @@ sub load_packages_cache {
 
 sub should_start_extra_nonfree {
     my $pkg = shift;
-    my ($arch, $component, $pkgname) = split /:/, $pkg;
+    my ($arch, $component, $pkgname, $pkgsize) = split /:/, $pkg;
 
 	if ( ($component eq "non-free") && $extranonfree) {
 		$extranonfree = 0; # Flag that we don't need to start new next time!
@@ -359,7 +359,7 @@ sub should_start_extra_nonfree {
 
 sub should_exclude_package {
     my $pkg = shift;
-    my ($arch, $component, $pkgname) = split /:/, $pkg;
+    my ($arch, $component, $pkgname, $pkgsize) = split /:/, $pkg;
     my $should_exclude = 0;
 
     foreach my $entry (@exclude_packages) {
@@ -991,7 +991,7 @@ sub add_packages {
     }
 
     my $pkg = shift;
-	my ($arch, $component, $pkgname) = split /:/, $pkg;
+	my ($arch, $component, $pkgname, $pkgsize) = split /:/, $pkg;
 
     if ("$arch" eq "" or "$pkgname" eq "" or "$pkgname" eq "") {
         die "inconsistent data passed to add_packages: $pkg\n";

tools/merge_package_lists

@@ -59,7 +59,7 @@ done
 echo "# Starting merge at "`date` >> $BDIR/list.mid
 if [ $source = "yes" ] ; then
 	echo "DONEDONE" > $BDIR/DONE
-	awk '
+	awk -v max_size=$MAX_PKG_SIZE '
 
 	# First parse the Sources files
 	/^Package:/ {
@@ -77,6 +77,17 @@ if [ $source = "yes" ] ; then
 				  }
 				}
 
+    /^Files:/   {
+                  in_files = 1
+                  next
+                }
+    /^ / {
+                  if (in_files) {
+                      size[srcname]+=$2
+                      next
+                  }
+                }
+
     # Done reading the Sources
 	/^DONEDONE/ {
 				  parsed=1
@@ -86,11 +97,13 @@ if [ $source = "yes" ] ; then
 
 	# Now start placing source packages, depending on the order of the binary packages
 	/.*/        {
+                  in_files = 0
 				  if (parsed) {
 				  	split($0,fields,":")
 					arch=fields[1]
 					component=fields[2]
 					pkg=fields[3]
+                    pkgsize=fields[4]
 					source=bin2src[pkg]
                     if ("" == source) {
       		            if ("main-installer" == component) {
@@ -123,13 +136,21 @@ if [ $source = "yes" ] ; then
 						}
 					    }
 					}
-					printf("%d:%s\n", FNR, $0)
+                    if (size[source] <= max_size) {
+                        printf("%d:%s\n", FNR, $0)
+                    } else {
+                        printf("%d:%s-DEPTOOBIG\n", FNR, $0)
+                    }
 				  }
 				}
 
 	END {
 	    for (source in included) {
-		printf("%d:aaaaaaaaaaaaaaaaaa:%s:%s\n", indexnr[source], sourcecomp[source], source)
+            if (size[source] > max_size) {
+                 printf("%d:aaaaaaaaaaaaaaaaaa:%s:%s-SRCTOOBIG\n", indexnr[source], sourcecomp[source], source, size[source])
+            } else {
+                 printf("%d:aaaaaaaaaaaaaaaaaa:%s:%s\n", indexnr[source], sourcecomp[source], source, size[source])
+            }
 	    }
 	}' $ADIR/$CODENAME-source/apt-state/lists/*Sources $BDIR/DONE $list | sort -nk 1 -t : >> $BDIR/list.mid
 else
@@ -141,7 +162,7 @@ echo "# Ending merge pass 1 at "`date` >> $BDIR/list.mid
 # prepended to allow us to shuffle them properly. Remove any comments
 # logged from the awk code above, then sort and remove the line numbers
 # again
-grep -v ^# $BDIR/list.mid | awk -F : '
+grep -v -e ^# -e TOOBIG $BDIR/list.mid | awk -F : '
     {
         if (!($4 in add))
         {
@@ -149,7 +170,7 @@ grep -v ^# $BDIR/list.mid | awk -F : '
         }
 		gsub("aaaaaaaaaaaaaaaaaa", "source", $2) # Undo the source sorting hack
 		gsub("main-installer", "main", $3)
-        add[$4] = add[$4] $2 ":" $3 ":" $4 " "
+        add[$4] = add[$4] $2 ":" $3 ":" $4 ":" $5 " "
     }
 END {
         for (i = 0; i < number_packages; i++)
@@ -160,3 +181,11 @@ END {
         }
     }' > $OUT
 echo "# Ending merge pass 2 at "`date` >> $BDIR/list.mid
+
+awk -F : '
+    /SRCTOOBIG/   { print $3 }
+' $BDIR/list.mid > $BDIR/sourcetoobig
+
+awk -F : '
+    /DEPTOOBIG/   { print $3 }
+' $BDIR/list.mid > $BDIR/sourcetoobig-deps

tools/sort_deps

@@ -294,7 +294,8 @@ open(CDLIST, "> $dir/packages.$arch")
     || die "Can't write in $dir/$_.packages.$arch: $!\n";
 foreach (@output) {
     my $component = $packages{$_}{"Component"};
-    print CDLIST "$arch:$component:$_\n";
+    my $size = $packages{$_}{"Size"};
+    print CDLIST "$arch:$component:$_:$size\n";
     $count++;
 }
 close CDLIST;

tools/start_new_disc

@@ -255,10 +255,26 @@ if [ $DISKNUM = 1 ] ; then
 	done
 
     # See if we had to remove anything because it was too big. Warn the user.
+    if [ -s $BDIR/sourcetoobig ] ; then
+        echo "  Adding README.excluded (some source files were too big)"
+        echo "For size reasons, the following source packages were excluded from this disc set:" >> $CDDIR/README.excluded
+        echo >> $CDDIR/README.excluded
+        cat $BDIR/sourcetoobig >> $CDDIR/README.excluded
+        if [ -s $BDIR/sourcetoobig-deps ] ; then
+            echo "and that caused the following packages to be also excluded because" >> $CDDIR/README.excluded
+            echo "of dependencies:" >> $CDDIR/README.excluded
+            echo >> $CDDIR/README.excluded
+            cat $BDIR/sourcetoobig-deps >> $CDDIR/README.excluded
+            echo >> $CDDIR/README.excluded
+            echo "Beware that this may break dependencies in this disc set." >> $CDDIR/README.excluded
+        fi
+        echo >> $CDDIR/README.excluded
+    fi
+
     grep "refused because of toobig" $BDIR/log.sort_deps* | sort -u | awk '{print $1}' > $BDIR/toobig
     if [ -s $BDIR/toobig ] ; then
-        echo "  Adding README.excluded (some files were too big)"
+        echo "  Adding README.excluded (some package files were too big)"
-        echo "For size reasons, the following packages were excluded from this disc set:" > $CDDIR/README.excluded
+        echo "For size reasons, the following packages were excluded from this disc set:" >> $CDDIR/README.excluded
         echo >> $CDDIR/README.excluded
         cat $BDIR/toobig >> $CDDIR/README.excluded
         grep "Can't add.*toobig" $BDIR/log.sort_deps* | sort -u | awk '{print $3}' > $BDIR/toobig-deps