Add non-free-firmware support

List the desired non-free sections to use in NONFREE_COMPONENTS in CONF.sh, default to "non-free non-free-firmware" by default until things are all migrated.
2022-10-08 17:45:11 +01:00 · 2022-10-08 17:45:11 +01:00 · e57aa6fa72
parent 8c52820262
commit e57aa6fa72
12 changed files with 85 additions and 35 deletions
--- a/CONF.sh
+++ b/CONF.sh
@ -132,20 +132,23 @@ export OUT=/srv/mirror/debian-cd-test
 # This cannot reside on an NFS mount.
 export APTTMP=/srv/mirror/tmp/apt

-# Do I want to have NONFREE merged in the CD set
+# Do I want to have NONFREE merged in the CD set?
 # export NONFREE=1

-# Do I want to have CONTRIB merged in the CD set
+# Do I want to have CONTRIB merged in the CD set?
 export CONTRIB=1

-# Do I want to have NONFREE on a separate CD (the last CD of the CD set)
+# Where should I look for non-free packages?
+export NONFREE_COMPONENTS="non-free non-free-firmware"
+
+# Do I want to have NONFREE on separate image(s) at the end set?
 # WARNING: Don't use NONFREE and EXTRANONFREE at the same time !
 # export EXTRANONFREE=1

 # Do I want to force (potentially non-free) firmware packages to be
 # placed on disc 1? Will make installation much easier if systems
 # contain hardware that depends on this firmware
-# export FORCE_FIRMWARE=1
+export FORCE_FIRMWARE=1

 # If you have a $MIRROR/dists/$CODENAME/local/binary-$ARCH dir with 
 # local packages that you want to put on the CD set then
--- a/15
+++ b/15
@ -133,12 +133,15 @@ The process of building a CD is composed of the following steps:

  You can change the behaviour of this command with the following
  variables:
-  - if NONFREE is set, then packages from non-free will be allowed
-    (NONFREE must be exported to all sub-shells)
-  - if EXTRANONFREE is set, then non-free packages will be included
-    on an extra CD (the last CD in fact). Don't use NONFREE and
-    EXTRANONFREE at the same time!
-    (EXTRANONFREE must be exported to all sub-shells)
+
+  - if NONFREE is set, then packages from the components defined in
+    NONFREE_COMPONENTS will be allowed (NONFREE must be exported to
+    all sub-shells)
+  - if EXTRANONFREE is set, then similarly non-free packages will be
+    included on an extra CD (the last CD in fact). Don't use NONFREE
+    and EXTRANONFREE at the same time! (EXTRANONFREE must be exported
+    to all sub-shells)
+  - if FORCE_FIRMWARE=1
  - if COMPLETE is set, all packages that are not listed in the
    selected task file will be included at the end
  - setting INSTALLER_CD will use an appropriate task file for
--- a/debian/changelog
+++ b/debian/changelog
@ -8,6 +8,9 @@ debian-cd (3.1.36) UNRELEASED; urgency=medium
  * Tweak grub boot menu highlight graphics to work again - switch
    from greyscale to colour, as newer grub has removed greyscale PNG
    support.. Closes: #1013079, #1013432.
+  * Add non-free-firmware support
+    + Add an extra config variable NONFREE_COMPONENTS to declare where
+      to look for non-free packages.

  [ Cyril Brulebois ]
  * generate_firmware_patterns: Drop support for the temporary --test
--- a/tools/apt-selection
+++ b/tools/apt-selection
@ -46,7 +46,7 @@ options=" -q -o Dir::State::status=$APTTMP/$THIS_PKGSET/status \

 sections=main
 if [ "${NONFREE:-0}" != "0" ] || [ "${EXTRANONFREE:-0}" != "0" ] || [ "${FORCE_FIRMWARE:-0}" != "0" ]; then
-	sections="$sections non-free"
+	sections="$sections $NONFREE_COMPONENTS"
 fi
 if [ "${CONTRIB:-0}" != "0" ]; then
 	sections="$sections contrib"
--- a/tools/create_control
+++ b/tools/create_control
@ -31,7 +31,7 @@ PREFIX=`echo $CDDIR | sed "s?$BDIR/CD?$BDIR/?"`

 NUM=${PREFIX##$BDIR/}
 if [ -n "$NONFREE" -o -n "$EXTRANONFREE" ]; then
-    SECTIONS="main contrib non-free"
+    SECTIONS="main contrib $NONFREE_COMPONENTS"
 else
    SECTIONS="main contrib"
 fi
--- a/tools/generate_di_list
+++ b/tools/generate_di_list
@ -75,7 +75,9 @@ my @common_excludes = read_exclude("exclude-udebs");
 my $mirror_path = "$ENV{MIRROR}/dists/$ENV{DI_CODENAME}";
 my @components = qw(main);
 push @components, 'contrib' if $ENV{CONTRIB};
-push @components, 'non-free' if $ENV{NONFREE};
+if ($ENV{NONFREE}) {
+    push @components, split /\ /,$ENV{NONFREE_COMPONENTS};
+}
 push @components, 'unreleased' if $ENV{UNRELEASED};
 push @components, 'local' if $ENV{LOCAL};

--- a/tools/grab_source_list
+++ b/tools/grab_source_list
@ -17,6 +17,11 @@ awk -v max_size=$MAX_PKG_SIZE '
 				  next
 				}

+	/^Section: non-free-firmware/ {
+				  component[srcname]="non-free-firmware"
+				  next
+				}
+
 	/^Section: contrib/ {
 				  component[srcname]="contrib"
 				  next
@ -58,11 +63,16 @@ awk -v max_size=$MAX_PKG_SIZE '
 if [ "$NONFREE"x = "1"x ] ; then
    grep -v SRCTOOBIG $BDIR/list.mid > $OUT
 else
-    grep -v -e :non-free: -e SRCTOOBIG $BDIR/list.mid > $OUT
+    grep -v \
+	 -e :non-free: \
+	 -e :non-free-firmware: \
+	 -e SRCTOOBIG $BDIR/list.mid > $OUT
 fi

 if [ "$EXTRANONFREE"x = "1"x ] ; then 
-	grep :non-free: $BDIR/list.mid | grep -v SRCTOOBIG >> $OUT
+	for COMPONENT in $NONFREE_COMPONENTS; do
+		grep :"$COMPONENT": $BDIR/list.mid | grep -v SRCTOOBIG >> $OUT
+	done
 fi

 awk -F : '
--- a/tools/make_disc_trees.pl
+++ b/tools/make_disc_trees.pl
@ -19,7 +19,7 @@ my ($basedir, $mirror, $tdir, $codename, $archlist, $mkisofs, $maxcds,
 my $mkisofs_base_opts = "";
 my $mkisofs_opts = "";
 my $mkisofs_dirs = "";
-my (@arches, @arches_nosrc, @overflowlist, @pkgs_added);
+my (@arches, @arches_nosrc, @overflowlist, @pkgs_added, @nonfree_components);
 my (@exclude_packages, @unexclude_packages, @excluded_package_list);
 my %firmware_package;
 my $current_checksum_type = "";
@ -88,6 +88,7 @@ if ($maxisos < $maxcds) {
 }

 $extranonfree = read_env('EXTRANONFREE', 0);
+@nonfree_components = split /\ /, read_env('NONFREE_COMPONENTS', 'non-free');
 $nonfree = read_env('NONFREE', 0);
 $contrib = read_env('CONTRIB', 0);
 $use_local = read_env('LOCAL', 0);
@ -265,7 +266,7 @@ while (defined (my $pkg = <INLIST>)) {
    if (should_exclude_package($pkg)) {
        push(@excluded_package_list, $pkg);
    } elsif (should_start_extra_nonfree($pkg)) {
-        print LOG "Starting on extra non-free CDs\n";
+        print LOG "Starting on extra non-free image(s)\n";
        finish_disc($cddir, "");
        # And reset, to start the next disc
        $size = 0;
@ -417,7 +418,9 @@ sub load_all_descriptions {
 	    load_descriptions("contrib", $use_backports);
 	}
 	if ($nonfree || $extranonfree) {
-	    load_descriptions("non-free", $use_backports);
+	    foreach my $component (@nonfree_components) {
+		load_descriptions($component, $use_backports);
+	    }
 	}
 	if ($use_local) {
 	    load_descriptions("local", $use_backports);
@ -490,11 +493,14 @@ sub should_start_extra_nonfree {
    my $pkg = shift;
    my ($arch, $component, $pkgname, $pkgsize) = split /:/, $pkg;

-	if ( ($component eq "non-free") && $extranonfree) {
+    if ($extranonfree) {
+	foreach my $nf_comp (@nonfree_components) {
+	    if ($component eq $nf_comp) {
 		$extranonfree = 0; # Flag that we don't need to start new next time!
 		return 1;
 	    }
-	
+	}
+    }
    return 0;
 }

@ -967,6 +973,8 @@ sub Packages_dir {
        $dist = "contrib";
    } elsif ($file =~ /\/non-free\//) {
        $dist = "non-free";
+    } elsif ($file =~ /\/non-free-firmware\//) {
+        $dist = "non-free-firmware";
    } else {
        $dist = "local";
    }	
@ -1186,6 +1194,8 @@ sub add_firmware_stuff {
        $dist = "contrib";
    } elsif ($file =~ /\/non-free\//) {
        $dist = "non-free";
+    } elsif ($file =~ /\/non-free-firmware\//) {
+        $dist = "non-free-firmware";
    } else {
        $dist = "local";
    }
--- a/tools/sort_deps
+++ b/tools/sort_deps
@ -22,6 +22,7 @@ my $listfile = shift;
 my $nonfree = read_env('NONFREE', 0);
 my $extranonfree = read_env('EXTRANONFREE', 0);
 my $force_firmware = read_env('FORCE_FIRMWARE', 0);
+my @nonfree_components = split /\ /, read_env('NONFREE_COMPONENTS', 'non-free');
 my $local = read_env('LOCAL', 0);
 my $complete = read_env('COMPLETE', 0);
 my $add_rec = ! read_env('NORECOMMENDS', 1);
@ -85,6 +86,10 @@ msg(1, "Complete selected packages with all the rest: ");
 msg(1, yesno($complete)."\n");
 msg(1, "Include non-free packages: ");
 msg(1, yesno($nonfree)."\n");
+msg(1, "Non-free components:\n");
+foreach my $cmp (@nonfree_components) {
+    msg(1, "  - $cmp\n");
+}
 msg(1, "Force inclusion of firmware packages: ");
 msg(1, yesno($force_firmware)."\n");
 msg(1, "Include Recommends: ");
@ -154,16 +159,20 @@ my $count_excl = 0;

 # Now exclude packages because of the non-free rules
 if (not $nonfree) {
-    foreach (grep { $packages{$_}{"Section"} =~ /non-free/ }
+    foreach my $cmp (@nonfree_components) {
+	msg(1, "Checking for $cmp packages to exclude\n");
+	foreach (grep { $packages{$_}{"Section"} =~ /$cmp\//}
 		 (keys %packages)) {
 	    if ($force_firmware and $packages{$_}{"IsFirmware"}) {
-            msg(1, "force_firmware: keeping non-free package $_\n");
+		msg(1, " - force_firmware: keeping $cmp package $_\n");
 	    } else {
+		msg(1, " - excluding $cmp package $_\n");
 		$excluded{$_} = 'nonfree';
 		$count_excl++;
 	    }
 	}
    }
+}

 msg(1, "Statistics:
 Number of packages: @{ [scalar(keys %packages)] }
@ -335,7 +344,7 @@ if ($extranonfree and (! $nonfree))
 {
 	my ($p, @toinclude);
 	
-	msg(0, "  Adding non-free packages now\n");
+	msg(0, "  Adding non-free* packages now\n");

 	# Finally accept non-free packages ...
 	foreach $p (grep { $excluded{$_} eq "nonfree" } (keys %excluded))
@ -466,6 +475,8 @@ sub parse_package {
 		$packages{$p}{"Component"} = "contrib";
 	} elsif ($packages{$p}{"Section"} =~ /non-free\//) {
 		$packages{$p}{"Component"} = "non-free";
+	} elsif ($packages{$p}{"Section"} =~ /non-free-firmware\//) {
+		$packages{$p}{"Component"} = "non-free-firmware";
 	} elsif ($packages{$p}{"IsUdeb"}) {
 		$packages{$p}{"Component"} = "main-installer";
 	} else {
--- a/tools/start_new_disc
+++ b/tools/start_new_disc
@ -343,7 +343,7 @@ cd $CDDIR

 SECTIONS="main"
 if [ "${NONFREE:-0}" != "0" -o "${EXTRANONFREE:-0}" != "0" ] ; then
-    SECTIONS="${SECTIONS} non-free"
+    SECTIONS="${SECTIONS} $NONFREE_COMPONENTS"
 fi
 if [ "${CONTRIB:-0}" != "0" ] ; then
    SECTIONS="${SECTIONS} contrib"
--- a/tools/which_deb
+++ b/tools/which_deb
@ -19,7 +19,9 @@ $text_out = "";

 my @components = qw(main);
 push @components, 'contrib' if $ENV{CONTRIB};
-push @components, 'non-free' if $ENV{NONFREE};
+if ($ENV{NONFREE}) {
+    push @components, split /\ /,$ENV{NONFREE_COMPONENTS};
+}
 push @components, 'local' if $ENV{LOCAL};

 if (!defined ($output)) {
--- a/12
+++ b/12
@ -16,6 +16,7 @@ MIRROR_NORM=/srv/cdbuilder.debian.org/src/ftp/debian

 # Do you want non-free? 1 for yes, 0 for no
 NONFREE=0
+NONFREE_FIRMWARE=1

 # What release version is this?
 VER=9.1.0
@ -81,7 +82,7 @@ CLOG=dists/$CODENAME/ChangeLog
 UPD=$TDIR/$CODENAME-update
 DATE=`date +%Y%m%d`
 BASEDIR=`pwd`
-SECTS="main non-free contrib"
+SECTS="main non-free non-free-firmware contrib"
 VERBOSE=2
 GRAB_CHECKSUMS=$BASEDIR/tools/grab_checksums
 export VERBOSE BASEDIR SECTS
@ -95,7 +96,7 @@ if [ "$ARCHLIST"x = ""x ] ; then
    ARCHLIST="arm64 armel armhf amd64 i386 mips mipsel mips64el ppc64el s390x source" # amd64 # -all dealt with specially
 fi

-export TDIR NONFREE VER MIRROR CODENAME OUT BASEDIR
+export TDIR NONFREE NONFREE_FIRMWARE VER MIRROR CODENAME OUT BASEDIR

 CDSIZE=$(($CDSIZE * 1024))

@ -278,7 +279,12 @@ fi
 cp $DIFF $UPD/list
 if [ "$NONFREE"x != "1"x ] ; then
    echo "Removing non-free files from the list"
-    grep -v non-free $UPD/list > $UPD/list1
+    grep -v non-free/ $UPD/list > $UPD/list1
+    mv -f $UPD/list1 $UPD/list
+fi
+if [ "$NONFREE_FIRMWARE"x != "1"x ] ; then
+    echo "Removing non-free-firmware files from the list"
+    grep -v non-free-firmware/ $UPD/list > $UPD/list1
    mv -f $UPD/list1 $UPD/list
 fi