* MAJOR CHANGE: Stop using apt-ftparchive to generate the Packages

files on each CD/DVD; instead grab the details for each package
  directly from the Packages file in the archive. MUCH MUCH faster,
  and uses less code too. Affected scripts:

  + tools/scanpackages now just generates Release files. argv[1] used
    to be "scan" or "install", but no longer used

  + tools/add_debs slightly simplified - old code removed

  + tools/add_packages now dumps out Packages file fragments as each
    .deb and .udeb is copied into the temporary trees

  + WARNING: Any uses of the "before-scanpackages" hook may now break
    - shout if you're affected.

debian/changelog

@@ -93,6 +93,22 @@ debian-cd (2.2.24) UNRELEASED; urgency=low
   * Split out the code that copies sources into the temporary trees and
     creates Sources files from the Makefile into tools/add_source_packages
 
+  * MAJOR CHANGE: Stop using apt-ftparchive to generate the Packages
+	files on each CD/DVD; instead grab the details for each package
+	directly from the Packages file in the archive. MUCH MUCH faster,
+	and uses less code too. Affected scripts:
+
+    + tools/scanpackages now just generates Release files. argv[1]
+      used to be "scan" or "install", but no longer used
+
+	+ tools/add_debs slightly simplified - old code removed
+
+    + tools/add_packages now dumps out Packages file fragments as each
+      .deb and .udeb is copied into the temporary trees
+
+    + WARNING: Any uses of the "before-scanpackages" hook may now
+      break - shout if you're affected.
+ 
  -- Joey Hess <joeyh@debian.org>  Thu,  8 Dec 2005 22:53:16 +0000
 
 debian-cd (2.2.23) unstable; urgency=low

tools/add_debs

@@ -92,33 +92,38 @@ do
     fi
     if [ -n "$BASE_EXCLUDE" ] ; then
 		if [ -r "$BASE_EXCLUDE" ] ; then
-			cp -af $BASE_EXCLUDE $BDIR/CD$DISK/.disk/base_exclude
+			cp -af "$BASE_EXCLUDE" "$BDIR/CD$DISK/.disk/base_exclude"
 		else
 			echo "ERROR: Unable to read BASE_EXCLUDE file $BASE_EXCLUDE"
 		fi
 	fi
 done
 
-# Generate the Packages files
+# Copy the debs in place, and generate Packages files as we go
 for i in $BDIR/*.packages
 do
 	dir=${i%%.packages}
 	n=${dir##$BDIR/}
 	dir=$BDIR/CD$n
 	echo "$n ... "
-  	cat $i | xargs -n 200 -r $ADDPACKAGES $dir
+
+	# Copy the deb files in; addpackages also creates the Packages
+	# files as it runs
+  	sort $i | xargs -n 200 -r $ADDPACKAGES $dir
 	if [ -x "$HOOK" ] ; then
 	   cd $BDIR && $HOOK $n before-scanpackages
 	fi
-	$SCANPACKAGES scan $dir
-	echo "done."
+
+	# Create the Packages.gz files. Should we also make Packages.bz2?
+	for file in `find $dir -name Packages`
+	do
+		echo "  $file.gz"
+		gzip -9 < $file > $file.gz
 	done
 
-# Now install the Packages and Packages.cd files
-for i in $BDIR/*.packages
-do
-	dir=${i%%.packages}
-	dir=${dir##$BDIR/}
-	dir=$BDIR/CD$dir
-	$SCANPACKAGES install $dir
+	# Now the Release files; has to be done last, as they contain the
+	# md5sums of the Packages and Packages.gz files
+	$SCANPACKAGES $dir
+
+	echo "done."
 done

tools/add_packages

@@ -13,6 +13,7 @@ my $localdebs = $ENV{'LOCALDEBS'} || $mirror;
 my $security = $ENV{'SECURITY'} || $mirror;
 my $nonus   = $ENV{'NONUS'}   || '';
 my $basedir = $ENV{'BASEDIR'} || die "Set the BASEDIR var ...\n";
+my $codename = $ENV{'CODENAME'} || die "Set the CODENAME var ...\n";
 
 require "$basedir/tools/link.pl";
 
@@ -21,13 +22,35 @@ open (LIST, "$basedir/tools/apt-selection cache show @ARGV |")
 
 $/ = ''; # Browse by paragraph
 
-my ($p, $file, $arch, $d, $realfile, $source, $section, $name);
+my ($p, $file, $arch, $d, $realfile, $source, $section, $name, $pkgfile, $dist, $pdir);
 while (defined($_ = <LIST>)) {
 	m/^Package: (\S+)/m and $p = $1;
 	m/^Filename: (\S+)/mi and $file = $1;
 	m/^Architecture: (\S+)/m and $arch = $1;
 	m/^Section: (\S+)/m and $section = $1;
 
+	if ($file =~ /main/) {
+		$dist = "main";
+	} elsif ($file =~ /contrib/) {
+		$dist = "contrib";
+	} elsif ($file =~ /non-free/) {
+		$dist = "non-free";
+	}
+
+	$pdir = "$dir/dists/$codename/$dist/binary-$ENV{'ARCH'}";
+	if ($section eq "debian-installer") {
+		$pdir = "$dir/dists/$codename/$dist/debian-installer/binary-$ENV{'ARCH'}";
+	}
+	$pkgfile = "$pdir/Packages";
+	
+	if (! -d $pdir) {
+		system("mkdir -p $pdir");
+	}	
+	
+	open(PFILE, ">>$pkgfile");
+	print PFILE $_;
+	close(PFILE);
+
 	$source = ($section =~ /non-US/i) ? $nonus : $mirror;
 
 	# This is a hack to allow the local debs to be located elsewhere.
@@ -44,7 +67,7 @@ while (defined($_ = <LIST>)) {
 	    
 	    # Check that the directory where the link will be created does
 	    # exist
-	    my $pdir = "$dir/$file";
+	    $pdir = "$dir/$file";
 	    $pdir =~ s#[^/]+$##g;
 	    if (! -d $pdir)
 	    {

tools/make_image

@@ -48,6 +48,8 @@ do
 	# Clean up any old files
 	rm -f $OUT/$OUTFILE.raw $OUT/$OUTFILE.jigdo $OUT/$OUTFILE.template
 
+	date
+
 	# Actually make the ISO/jigdo images. Long command lines
 	# here... :-(
 	case $DOJIGDO in

tools/scanpackages

@@ -3,9 +3,8 @@
 # Copyright 1999 Raphaël Hertzog <hertzog@debian.org>
 # See the README file for the license
 
-# This script will create the Packages.gz and the Packages.cd files
-# First arg = "scan" or "install"
-# Second arg = directory of the CD
+# This script will create the Release files
+# First arg = directory of the CD
 
 #set -e
 
@@ -28,11 +27,7 @@ if [ -n "$LOCAL" ]; then
 	SECTIONS="$SECTIONS local"
 fi
 
-
-
-cd "$2"
-
-if [ "$1" = "install" ]; then
+cd "$1"
 
 # these two lines can be inserted below the "Architectures" line below,
 # to sort out broken Release files just after the testing --> stable
@@ -54,6 +49,7 @@ else
 	echo "ERROR: Release file ($MIRROR/dists/$CODENAME/Release) is missing !"
 	exit 1
 fi
+
 if [ -n "$NONUS" ] ; then
 	if [ -e "$NONUS/dists/$CODENAME/non-US/Release" ] ; then
 		# Strip the MD5Sum and SHA1 fields
@@ -71,8 +67,10 @@ else
 		exit 1
 	fi
 fi
+
 # Copying release files
-for SECT in $SECTIONS; do
+for SECT in $SECTIONS
+do
 	# Install the release files
 	if [ -e "$MIRROR/dists/$CODENAME/$SECT/binary-$ARCH/Release" ] ; then
 		cp $MIRROR/dists/$CODENAME/$SECT/binary-$ARCH/Release \
@@ -96,152 +94,8 @@ if [ -e "$MIRROR/dists/$DI_CODENAME/main/debian-installer/binary-$ARCH/Release"
 	   dists/$CODENAME/main/debian-installer/binary-$ARCH/
 fi
 
-# Generating Packages files
-apt-ftparchive generate $PREFIX.generate-binary
-if [ -n "$NONUS" ]; then
-    apt-ftparchive generate $PREFIX.generate-binary-non-US
-fi
-if [ -e "$MIRROR/dists/$DI_CODENAME/main/debian-installer" ]; then
-    apt-ftparchive generate $PREFIX.generate-binary-debian-installer
-fi
-
-fi ## END "$1" = "install"
-
-# Prepare the files needed to generate the Packages files
-if [ "$1" != "scan" ]; then exit 0; fi
-
-#
-# Create the file lists and the config files for apt-ftparchive generate
-#
-
-# We have to scan all possible dists where packages can be
-DISTS=""
-DISTSNONUS=""
-for i in `cd dists; echo *; cd ..`; do
-    if [ ! -L "dists/$i" -a -d "dists/$i" ]; then
-	if [ -d "dists/$i/main/binary-$ARCH" -o \
-	     -d "dists/$i/non-free/binary-$ARCH" -o \
-	     -d "dists/$i/contrib/binary-$ARCH" ]; then
-	    DISTS="$DISTS $i"
-	fi
-        if [ -n "$SECURITY" ]; then
-            DISTS="$DISTS $i/updates"
-        fi
-	if [ -d "dists/$i/non-US/main/binary-$ARCH" -o \
-	     -d "dists/$i/non-US/non-free/binary-$ARCH" -o \
-	     -d "dists/$i/non-US/contrib/binary-$ARCH" ]; then
-	    DISTSNONUS="$DISTSNONUS $i"
-	fi
-    fi
-done
-# Creating the file lists
-for SECT in $SECTIONS; do
-    touch $PREFIX.filelist_$SECT
-    if [ -d "pool/$SECT" ]; then
-	find pool/$SECT >>$PREFIX.filelist_$SECT
-    fi
-    if [ -d "pool/unstable/$SECT" ]; then
-	find pool/unstable/$SECT >>$PREFIX.filelist_$SECT
-    fi
-    if [ -n "$SECURITY" -a -d "pool/updates/$SECT" ]; then
-	find pool/updates/$SECT >>$PREFIX.filelist_$SECT
-    fi
-    for DIST in $DISTS; do
-	if [ -d "dists/$DIST/$SECT/binary-$ARCH" ]; then
-	    find dists/$DIST/$SECT/binary-$ARCH >>$PREFIX.filelist_$SECT
-	fi
-    done
-    if [ -n "$NONUS" ]; then
-	touch $PREFIX.filelist_non-US_$SECT
-	if [ -d "pool/non-US/$SECT" ]; then
-	    find pool/non-US/$SECT >>$PREFIX.filelist_non-US_$SECT
-	fi
-	for DIST in $DISTSNONUS; do
-	    if [ -d "dists/$DIST/non-US/$SECT/binary-$ARCH" ]; then
-		find dists/$DIST/non-US/$SECT/binary-$ARCH \
-		    >>$PREFIX.filelist_non-US_$SECT
-	    fi
-	done
-    fi
-done
-
-# Creating the config files
-
-# SAM 2005-11-01:
-# Nasty hack - we may need to look at extra override files from sid.
-# See if the extra override exists; if not redirect.
-EXTRA_CODENAME=$CODENAME
-if [ ! -e $BDIR/indices/override.$EXTRA_CODENAME.extra.main.gz ] ; then
-    EXTRA_CODENAME=sid
-fi
-cat >$PREFIX.generate-binary <<EOF
-Dir::ArchiveDir "$2";
-Dir::OverrideDir "$BDIR/indices";
-Dir::CacheDir "$APTTMP/$CODENAME-$ARCH/apt-ftparchive-db";
-
-TreeDefault::Contents " ";
-
-Tree "dists/$CODENAME" {
-    FileList "$PREFIX.filelist_\$(SECTION)";
-    Sections "$SECTIONS";
-    Architectures "$ARCH";
-    
-    BinOverride "override.$CODENAME.\$(SECTION)";
-    ExtraOverride "override.$EXTRA_CODENAME.extra.\$(SECTION)";
-    SrcOverride "override.$CODENAME.\$(SECTION).src";
-}
-EOF
-cat >$PREFIX.generate-binary-non-US <<EOF
-Dir::ArchiveDir "$2";
-Dir::OverrideDir "$BDIR/indices-non-US";
-Dir::CacheDir "$APTTMP/$CODENAME-$ARCH/apt-ftparchive-db";
-
-TreeDefault::Contents " ";
-
-Tree "dists/$CODENAME/non-US" {
-    FileList "$PREFIX.filelist_non-US_\$(SECTION)";
-    Sections "`echo $SECTIONSNONUS | sed -e 's#non-US/##g'`";
-    Architectures "$ARCH";
-    
-    BinOverride "override.$CODENAME.\$(SECTION)";
-    ExtraOverride "override.$EXTRA_CODENAME.extra.\$(SECTION)";
-    SrcOverride "override.$CODENAME.\$(SECTION).src";
-}
-EOF
-cat >$PREFIX.generate-binary-debian-installer <<EOF
-Dir::ArchiveDir "$2";
-Dir::OverrideDir "$BDIR/indices";
-Dir::CacheDir "$APTTMP/$CODENAME-$ARCH/apt-ftparchive-db";
-
-TreeDefault::Contents " ";
-
-Tree "dists/$CODENAME/main" {
-    FileList "$PREFIX.filelist_main";
-    Sections "debian-installer";
-    Architectures "$ARCH";
-    
-    BinOverride "override.$CODENAME.main.\$(SECTION)";
-    ExtraOverride "override.$EXTRA_CODENAME.extra.main";
-    SrcOverride "override.$CODENAME.main.src";
-    Packages::Extensions ".udeb";
-}
-EOF
-if [ -n "$LOCAL" -a -e "${LOCALDEBS:-$MIRROR}/dists/$DI_CODENAME/local/debian-installer" ]; then
-cat >>$PREFIX.generate-binary-debian-installer <<EOF
-
-Tree "dists/$CODENAME/local" {
-    FileList "$PREFIX.filelist_local";
-    Sections "debian-installer";
-    Architectures "$ARCH";
-    
-    BinOverride "override.$CODENAME.local.\$(SECTION)";
-    ExtraOverride "override.$EXTRA_CODENAME.extra.local";
-    SrcOverride "override.$CODENAME.local.src";
-    Packages::Extensions ".udeb";
-}
-EOF
-fi
-
+# Generating Packages files is now done in add_debs; we no longer use
+# apt-ftparchive
 
 # Creating the indices directory
 if [ ! -d "$BDIR/indices" ]; then