Add non-free-firmware support

List the desired non-free sections to use in NONFREE_COMPONENTS in
CONF.sh, default to "non-free non-free-firmware" by default until
things are all migrated.

CONF.sh

@@ -132,20 +132,23 @@ export OUT=/srv/mirror/debian-cd-test
 # This cannot reside on an NFS mount.
 export APTTMP=/srv/mirror/tmp/apt
 
-# Do I want to have NONFREE merged in the CD set
+# Do I want to have NONFREE merged in the CD set?
 # export NONFREE=1
 
-# Do I want to have CONTRIB merged in the CD set
+# Do I want to have CONTRIB merged in the CD set?
 export CONTRIB=1
 
-# Do I want to have NONFREE on a separate CD (the last CD of the CD set)
+# Where should I look for non-free packages?
+export NONFREE_COMPONENTS="non-free non-free-firmware"
+
+# Do I want to have NONFREE on separate image(s) at the end set?
 # WARNING: Don't use NONFREE and EXTRANONFREE at the same time !
 # export EXTRANONFREE=1
 
 # Do I want to force (potentially non-free) firmware packages to be
 # placed on disc 1? Will make installation much easier if systems
 # contain hardware that depends on this firmware
-# export FORCE_FIRMWARE=1
+export FORCE_FIRMWARE=1
 
 # If you have a $MIRROR/dists/$CODENAME/local/binary-$ARCH dir with 
 # local packages that you want to put on the CD set then

README

@@ -133,12 +133,15 @@ The process of building a CD is composed of the following steps:
 
   You can change the behaviour of this command with the following
   variables:
-  - if NONFREE is set, then packages from non-free will be allowed
+
-    (NONFREE must be exported to all sub-shells)
+  - if NONFREE is set, then packages from the components defined in
-  - if EXTRANONFREE is set, then non-free packages will be included
+    NONFREE_COMPONENTS will be allowed (NONFREE must be exported to
-    on an extra CD (the last CD in fact). Don't use NONFREE and
+    all sub-shells)
-    EXTRANONFREE at the same time!
+  - if EXTRANONFREE is set, then similarly non-free packages will be
-    (EXTRANONFREE must be exported to all sub-shells)
+    included on an extra CD (the last CD in fact). Don't use NONFREE
+    and EXTRANONFREE at the same time! (EXTRANONFREE must be exported
+    to all sub-shells)
+  - if FORCE_FIRMWARE=1
   - if COMPLETE is set, all packages that are not listed in the
     selected task file will be included at the end
   - setting INSTALLER_CD will use an appropriate task file for

debian/changelog

@@ -8,6 +8,9 @@ debian-cd (3.1.36) UNRELEASED; urgency=medium
   * Tweak grub boot menu highlight graphics to work again - switch
     from greyscale to colour, as newer grub has removed greyscale PNG
     support.. Closes: #1013079, #1013432.
+  * Add non-free-firmware support
+    + Add an extra config variable NONFREE_COMPONENTS to declare where
+      to look for non-free packages.
 
   [ Cyril Brulebois ]
   * generate_firmware_patterns: Drop support for the temporary --test

tools/apt-selection

@@ -46,7 +46,7 @@ options=" -q -o Dir::State::status=$APTTMP/$THIS_PKGSET/status \
 
 sections=main
 if [ "${NONFREE:-0}" != "0" ] || [ "${EXTRANONFREE:-0}" != "0" ] || [ "${FORCE_FIRMWARE:-0}" != "0" ]; then
-	sections="$sections non-free"
+	sections="$sections $NONFREE_COMPONENTS"
 fi
 if [ "${CONTRIB:-0}" != "0" ]; then
 	sections="$sections contrib"

tools/create_control

@@ -31,7 +31,7 @@ PREFIX=`echo $CDDIR | sed "s?$BDIR/CD?$BDIR/?"`
 
 NUM=${PREFIX##$BDIR/}
 if [ -n "$NONFREE" -o -n "$EXTRANONFREE" ]; then
-    SECTIONS="main contrib non-free"
+    SECTIONS="main contrib $NONFREE_COMPONENTS"
 else
     SECTIONS="main contrib"
 fi

tools/generate_di_list

@@ -75,7 +75,9 @@ my @common_excludes = read_exclude("exclude-udebs");
 my $mirror_path = "$ENV{MIRROR}/dists/$ENV{DI_CODENAME}";
 my @components = qw(main);
 push @components, 'contrib' if $ENV{CONTRIB};
-push @components, 'non-free' if $ENV{NONFREE};
+if ($ENV{NONFREE}) {
+    push @components, split /\ /,$ENV{NONFREE_COMPONENTS};
+}
 push @components, 'unreleased' if $ENV{UNRELEASED};
 push @components, 'local' if $ENV{LOCAL};
 

tools/grab_source_list

@@ -17,6 +17,11 @@ awk -v max_size=$MAX_PKG_SIZE '
 				  next
 				}
 
+	/^Section: non-free-firmware/ {
+				  component[srcname]="non-free-firmware"
+				  next
+				}
+
 	/^Section: contrib/ {
 				  component[srcname]="contrib"
 				  next
@@ -58,11 +63,16 @@ awk -v max_size=$MAX_PKG_SIZE '
 if [ "$NONFREE"x = "1"x ] ; then
     grep -v SRCTOOBIG $BDIR/list.mid > $OUT
 else
-    grep -v -e :non-free: -e SRCTOOBIG $BDIR/list.mid > $OUT
+    grep -v \
+	 -e :non-free: \
+	 -e :non-free-firmware: \
+	 -e SRCTOOBIG $BDIR/list.mid > $OUT
 fi
 
 if [ "$EXTRANONFREE"x = "1"x ] ; then 
-	grep :non-free: $BDIR/list.mid | grep -v SRCTOOBIG >> $OUT
+	for COMPONENT in $NONFREE_COMPONENTS; do
+		grep :"$COMPONENT": $BDIR/list.mid | grep -v SRCTOOBIG >> $OUT
+	done
 fi
 
 awk -F : '

tools/make_disc_trees.pl

@@ -19,7 +19,7 @@ my ($basedir, $mirror, $tdir, $codename, $archlist, $mkisofs, $maxcds,
 my $mkisofs_base_opts = "";
 my $mkisofs_opts = "";
 my $mkisofs_dirs = "";
-my (@arches, @arches_nosrc, @overflowlist, @pkgs_added);
+my (@arches, @arches_nosrc, @overflowlist, @pkgs_added, @nonfree_components);
 my (@exclude_packages, @unexclude_packages, @excluded_package_list);
 my %firmware_package;
 my $current_checksum_type = "";
@@ -88,6 +88,7 @@ if ($maxisos < $maxcds) {
 }
 
 $extranonfree = read_env('EXTRANONFREE', 0);
+@nonfree_components = split /\ /, read_env('NONFREE_COMPONENTS', 'non-free');
 $nonfree = read_env('NONFREE', 0);
 $contrib = read_env('CONTRIB', 0);
 $use_local = read_env('LOCAL', 0);
@@ -265,7 +266,7 @@ while (defined (my $pkg = <INLIST>)) {
     if (should_exclude_package($pkg)) {
         push(@excluded_package_list, $pkg);
     } elsif (should_start_extra_nonfree($pkg)) {
-        print LOG "Starting on extra non-free CDs\n";
+        print LOG "Starting on extra non-free image(s)\n";
         finish_disc($cddir, "");
         # And reset, to start the next disc
         $size = 0;
@@ -417,7 +418,9 @@ sub load_all_descriptions {
 	    load_descriptions("contrib", $use_backports);
 	}
 	if ($nonfree || $extranonfree) {
-	    load_descriptions("non-free", $use_backports);
+	    foreach my $component (@nonfree_components) {
+		load_descriptions($component, $use_backports);
+	    }
 	}
 	if ($use_local) {
 	    load_descriptions("local", $use_backports);
@@ -490,12 +493,15 @@ sub should_start_extra_nonfree {
     my $pkg = shift;
     my ($arch, $component, $pkgname, $pkgsize) = split /:/, $pkg;
 
-	if ( ($component eq "non-free") && $extranonfree) {
+    if ($extranonfree) {
+	foreach my $nf_comp (@nonfree_components) {
+	    if ($component eq $nf_comp) {
 		$extranonfree = 0; # Flag that we don't need to start new next time!
 		return 1;
+	    }
 	}
-	
+    }
-	return 0;
+    return 0;
 }
 
 sub should_exclude_package {
@@ -967,6 +973,8 @@ sub Packages_dir {
         $dist = "contrib";
     } elsif ($file =~ /\/non-free\//) {
         $dist = "non-free";
+    } elsif ($file =~ /\/non-free-firmware\//) {
+        $dist = "non-free-firmware";
     } else {
         $dist = "local";
     }	
@@ -1186,6 +1194,8 @@ sub add_firmware_stuff {
         $dist = "contrib";
     } elsif ($file =~ /\/non-free\//) {
         $dist = "non-free";
+    } elsif ($file =~ /\/non-free-firmware\//) {
+        $dist = "non-free-firmware";
     } else {
         $dist = "local";
     }

tools/sort_deps

@@ -22,6 +22,7 @@ my $listfile = shift;
 my $nonfree = read_env('NONFREE', 0);
 my $extranonfree = read_env('EXTRANONFREE', 0);
 my $force_firmware = read_env('FORCE_FIRMWARE', 0);
+my @nonfree_components = split /\ /, read_env('NONFREE_COMPONENTS', 'non-free');
 my $local = read_env('LOCAL', 0);
 my $complete = read_env('COMPLETE', 0);
 my $add_rec = ! read_env('NORECOMMENDS', 1);
@@ -85,6 +86,10 @@ msg(1, "Complete selected packages with all the rest: ");
 msg(1, yesno($complete)."\n");
 msg(1, "Include non-free packages: ");
 msg(1, yesno($nonfree)."\n");
+msg(1, "Non-free components:\n");
+foreach my $cmp (@nonfree_components) {
+    msg(1, "  - $cmp\n");
+}
 msg(1, "Force inclusion of firmware packages: ");
 msg(1, yesno($force_firmware)."\n");
 msg(1, "Include Recommends: ");
@@ -154,14 +159,18 @@ my $count_excl = 0;
 
 # Now exclude packages because of the non-free rules
 if (not $nonfree) {
-    foreach (grep { $packages{$_}{"Section"} =~ /non-free/ }
+    foreach my $cmp (@nonfree_components) {
-	     (keys %packages)) {
+	msg(1, "Checking for $cmp packages to exclude\n");
-        if ($force_firmware and $packages{$_}{"IsFirmware"}) {
+	foreach (grep { $packages{$_}{"Section"} =~ /$cmp\//}
-            msg(1, "force_firmware: keeping non-free package $_\n");
+		 (keys %packages)) {
-        } else {
+	    if ($force_firmware and $packages{$_}{"IsFirmware"}) {
-            $excluded{$_} = 'nonfree';
+		msg(1, " - force_firmware: keeping $cmp package $_\n");
-            $count_excl++;
+	    } else {
-        }
+		msg(1, " - excluding $cmp package $_\n");
+		$excluded{$_} = 'nonfree';
+		$count_excl++;
+	    }
+	}
     }
 }
 
@@ -335,7 +344,7 @@ if ($extranonfree and (! $nonfree))
 {
 	my ($p, @toinclude);
 	
-	msg(0, "  Adding non-free packages now\n");
+	msg(0, "  Adding non-free* packages now\n");
 
 	# Finally accept non-free packages ...
 	foreach $p (grep { $excluded{$_} eq "nonfree" } (keys %excluded))
@@ -466,6 +475,8 @@ sub parse_package {
 		$packages{$p}{"Component"} = "contrib";
 	} elsif ($packages{$p}{"Section"} =~ /non-free\//) {
 		$packages{$p}{"Component"} = "non-free";
+	} elsif ($packages{$p}{"Section"} =~ /non-free-firmware\//) {
+		$packages{$p}{"Component"} = "non-free-firmware";
 	} elsif ($packages{$p}{"IsUdeb"}) {
 		$packages{$p}{"Component"} = "main-installer";
 	} else {

tools/start_new_disc

@@ -343,7 +343,7 @@ cd $CDDIR
 
 SECTIONS="main"
 if [ "${NONFREE:-0}" != "0" -o "${EXTRANONFREE:-0}" != "0" ] ; then
-    SECTIONS="${SECTIONS} non-free"
+    SECTIONS="${SECTIONS} $NONFREE_COMPONENTS"
 fi
 if [ "${CONTRIB:-0}" != "0" ] ; then
     SECTIONS="${SECTIONS} contrib"

tools/which_deb

@@ -19,7 +19,9 @@ $text_out = "";
 
 my @components = qw(main);
 push @components, 'contrib' if $ENV{CONTRIB};
-push @components, 'non-free' if $ENV{NONFREE};
+if ($ENV{NONFREE}) {
+    push @components, split /\ /,$ENV{NONFREE_COMPONENTS};
+}
 push @components, 'local' if $ENV{LOCAL};
 
 if (!defined ($output)) {

update-cd

@@ -16,6 +16,7 @@ MIRROR_NORM=/srv/cdbuilder.debian.org/src/ftp/debian
 
 # Do you want non-free? 1 for yes, 0 for no
 NONFREE=0
+NONFREE_FIRMWARE=1
 
 # What release version is this?
 VER=9.1.0
@@ -81,7 +82,7 @@ CLOG=dists/$CODENAME/ChangeLog
 UPD=$TDIR/$CODENAME-update
 DATE=`date +%Y%m%d`
 BASEDIR=`pwd`
-SECTS="main non-free contrib"
+SECTS="main non-free non-free-firmware contrib"
 VERBOSE=2
 GRAB_CHECKSUMS=$BASEDIR/tools/grab_checksums
 export VERBOSE BASEDIR SECTS
@@ -95,7 +96,7 @@ if [ "$ARCHLIST"x = ""x ] ; then
     ARCHLIST="arm64 armel armhf amd64 i386 mips mipsel mips64el ppc64el s390x source" # amd64 # -all dealt with specially
 fi
 
-export TDIR NONFREE VER MIRROR CODENAME OUT BASEDIR
+export TDIR NONFREE NONFREE_FIRMWARE VER MIRROR CODENAME OUT BASEDIR
 
 CDSIZE=$(($CDSIZE * 1024))
 
@@ -278,7 +279,12 @@ fi
 cp $DIFF $UPD/list
 if [ "$NONFREE"x != "1"x ] ; then
     echo "Removing non-free files from the list"
-    grep -v non-free $UPD/list > $UPD/list1
+    grep -v non-free/ $UPD/list > $UPD/list1
+    mv -f $UPD/list1 $UPD/list
+fi
+if [ "$NONFREE_FIRMWARE"x != "1"x ] ; then
+    echo "Removing non-free-firmware files from the list"
+    grep -v non-free-firmware/ $UPD/list > $UPD/list1
     mv -f $UPD/list1 $UPD/list
 fi