* MAJOR CHANGE: Stop using apt-ftparchive to generate the Packages

files on each CD/DVD; instead grab the details for each package
  directly from the Packages file in the archive. MUCH MUCH faster,
  and uses less code too. Affected scripts:

  + tools/scanpackages now just generates Release files. argv[1] used
    to be "scan" or "install", but no longer used

  + tools/add_debs slightly simplified - old code removed

  + tools/add_packages now dumps out Packages file fragments as each
    .deb and .udeb is copied into the temporary trees

  + WARNING: Any uses of the "before-scanpackages" hook may now break
    - shout if you're affected.
This commit is contained in:
Steve McIntyre 2005-12-13 02:04:33 +00:00
parent b10be4a9dc
commit 2358dd635e
5 changed files with 135 additions and 235 deletions

16
debian/changelog vendored
View File

@ -92,6 +92,22 @@ debian-cd (2.2.24) UNRELEASED; urgency=low
creates Packages files from the Makefile into tools/add_debs
* Split out the code that copies sources into the temporary trees and
creates Sources files from the Makefile into tools/add_source_packages
* MAJOR CHANGE: Stop using apt-ftparchive to generate the Packages
files on each CD/DVD; instead grab the details for each package
directly from the Packages file in the archive. MUCH MUCH faster,
and uses less code too. Affected scripts:
+ tools/scanpackages now just generates Release files. argv[1]
used to be "scan" or "install", but no longer used
+ tools/add_debs slightly simplified - old code removed
+ tools/add_packages now dumps out Packages file fragments as each
.deb and .udeb is copied into the temporary trees
+ WARNING: Any uses of the "before-scanpackages" hook may now
break - shout if you're affected.
-- Joey Hess <joeyh@debian.org> Thu, 8 Dec 2005 22:53:16 +0000

View File

@ -92,33 +92,38 @@ do
fi
if [ -n "$BASE_EXCLUDE" ] ; then
if [ -r "$BASE_EXCLUDE" ] ; then
cp -af $BASE_EXCLUDE $BDIR/CD$DISK/.disk/base_exclude
cp -af "$BASE_EXCLUDE" "$BDIR/CD$DISK/.disk/base_exclude"
else
echo "ERROR: Unable to read BASE_EXCLUDE file $BASE_EXCLUDE"
fi
fi
done
# Generate the Packages files
# Copy the debs in place, and generate Packages files as we go
for i in $BDIR/*.packages
do
dir=${i%%.packages}
n=${dir##$BDIR/}
dir=$BDIR/CD$n
echo "$n ... "
cat $i | xargs -n 200 -r $ADDPACKAGES $dir
# Copy the deb files in; addpackages also creates the Packages
# files as it runs
sort $i | xargs -n 200 -r $ADDPACKAGES $dir
if [ -x "$HOOK" ] ; then
cd $BDIR && $HOOK $n before-scanpackages
fi
$SCANPACKAGES scan $dir
# Create the Packages.gz files. Should we also make Packages.bz2?
for file in `find $dir -name Packages`
do
echo " $file.gz"
gzip -9 < $file > $file.gz
done
# Now the Release files; has to be done last, as they contain the
# md5sums of the Packages and Packages.gz files
$SCANPACKAGES $dir
echo "done."
done
# Now install the Packages and Packages.cd files
for i in $BDIR/*.packages
do
dir=${i%%.packages}
dir=${dir##$BDIR/}
dir=$BDIR/CD$dir
$SCANPACKAGES install $dir
done

View File

@ -13,6 +13,7 @@ my $localdebs = $ENV{'LOCALDEBS'} || $mirror;
my $security = $ENV{'SECURITY'} || $mirror;
my $nonus = $ENV{'NONUS'} || '';
my $basedir = $ENV{'BASEDIR'} || die "Set the BASEDIR var ...\n";
my $codename = $ENV{'CODENAME'} || die "Set the CODENAME var ...\n";
require "$basedir/tools/link.pl";
@ -21,18 +22,40 @@ open (LIST, "$basedir/tools/apt-selection cache show @ARGV |")
$/ = ''; # Browse by paragraph
my ($p, $file, $arch, $d, $realfile, $source, $section, $name);
my ($p, $file, $arch, $d, $realfile, $source, $section, $name, $pkgfile, $dist, $pdir);
while (defined($_ = <LIST>)) {
m/^Package: (\S+)/m and $p = $1;
m/^Filename: (\S+)/mi and $file = $1;
m/^Architecture: (\S+)/m and $arch = $1;
m/^Section: (\S+)/m and $section = $1;
if ($file =~ /main/) {
$dist = "main";
} elsif ($file =~ /contrib/) {
$dist = "contrib";
} elsif ($file =~ /non-free/) {
$dist = "non-free";
}
$pdir = "$dir/dists/$codename/$dist/binary-$ENV{'ARCH'}";
if ($section eq "debian-installer") {
$pdir = "$dir/dists/$codename/$dist/debian-installer/binary-$ENV{'ARCH'}";
}
$pkgfile = "$pdir/Packages";
if (! -d $pdir) {
system("mkdir -p $pdir");
}
open(PFILE, ">>$pkgfile");
print PFILE $_;
close(PFILE);
$source = ($section =~ /non-US/i) ? $nonus : $mirror;
# This is a hack to allow the local debs to be located elsewhere.
$source=$localdebs if $file=~m:local/:;
$source=$security if $file=~m:updates/:;
$source=$security if $file=~m:updates/:;
# If arch=all and filename points to binary-all then create
# a symbolic link in binary-$ARCH or if arch=all and filename is a
@ -44,7 +67,7 @@ while (defined($_ = <LIST>)) {
# Check that the directory where the link will be created does
# exist
my $pdir = "$dir/$file";
$pdir = "$dir/$file";
$pdir =~ s#[^/]+$##g;
if (! -d $pdir)
{

View File

@ -48,6 +48,8 @@ do
# Clean up any old files
rm -f $OUT/$OUTFILE.raw $OUT/$OUTFILE.jigdo $OUT/$OUTFILE.template
date
# Actually make the ISO/jigdo images. Long command lines
# here... :-(
case $DOJIGDO in

View File

@ -3,9 +3,8 @@
# Copyright 1999 Raphaël Hertzog <hertzog@debian.org>
# See the README file for the license
# This script will create the Packages.gz and the Packages.cd files
# First arg = "scan" or "install"
# Second arg = directory of the CD
# This script will create the Release files
# First arg = directory of the CD
#set -e
@ -15,24 +14,20 @@ PREFIX=`echo $2 | sed "s?$BDIR/CD?$BDIR/?"`
SECTIONS="main"
SECTIONSNONUS="non-US/main"
if [ "${NONFREE:-0}" != "0" -o "${EXTRANONFREE:-0}" != "0" ]; then
SECTIONS="${SECTIONS} non-free"
SECTIONSNONUS="${SECTIONSNONUS} non-US/non-free"
if [ "${NONFREE:-0}" != "0" -o "${EXTRANONFREE:-0}" != "0" ] ; then
SECTIONS="${SECTIONS} non-free"
SECTIONSNONUS="${SECTIONSNONUS} non-US/non-free"
fi
if [ "${CONTRIB:-0}" != "0" ]; then
SECTIONS="${SECTIONS} contrib"
SECTIONSNONUS="${SECTIONSNONUS} non-US/contrib"
if [ "${CONTRIB:-0}" != "0" ] ; then
SECTIONS="${SECTIONS} contrib"
SECTIONSNONUS="${SECTIONSNONUS} non-US/contrib"
fi
if [ -n "$LOCAL" ]; then
SECTIONS="$SECTIONS local"
if [ -n "$LOCAL" ] ; then
SECTIONS="$SECTIONS local"
fi
cd "$2"
if [ "$1" = "install" ]; then
cd "$1"
# these two lines can be inserted below the "Architectures" line below,
# to sort out broken Release files just after the testing --> stable
@ -40,208 +35,67 @@ if [ "$1" = "install" ]; then
# sed -e "s/^Suite: .*$/Suite: stable/" | \
# sed -e "s/^Description: .*$/Description: Debian $DEBVERSION/" | \
if [ -e "$MIRROR/dists/$CODENAME/Release" ]; then
# Strip the MD5Sum and SHA1 fields
# Update some other information as well
sed -e "s/^Architectures: .*$/Architectures: $ARCH/" \
$MIRROR/dists/$CODENAME/Release | \
sed -e "s|^Components: .*$|Components: $SECTIONS|" | \
perl -ne 'if (/^(MD5Sum|SHA1):/i) { $f=1; next }
if ($f) {
unless (/^ /) { print; $f=0 }
} else { print }' > dists/$CODENAME/Release
if [ -e "$MIRROR/dists/$CODENAME/Release" ] ; then
# Strip the MD5Sum and SHA1 fields
# Update some other information as well
sed -e "s/^Architectures: .*$/Architectures: $ARCH/" \
$MIRROR/dists/$CODENAME/Release | \
sed -e "s|^Components: .*$|Components: $SECTIONS|" | \
perl -ne 'if (/^(MD5Sum|SHA1):/i) { $f=1; next }
if ($f) {
unless (/^ /) { print; $f=0 }
} else { print }' > dists/$CODENAME/Release
else
echo "ERROR: Release file ($MIRROR/dists/$CODENAME/Release) is missing !"
exit 1
fi
if [ -n "$NONUS" ]; then
if [ -e "$NONUS/dists/$CODENAME/non-US/Release" ]; then
# Strip the MD5Sum and SHA1 fields
# Update some other information as well
sed -e "/^ /d" -e "s/^Architectures: .*$/Architectures: $ARCH/" \
$NONUS/dists/$CODENAME/non-US/Release | \
sed -e "s|^Components: .*$|Components: $SECTIONSNONUS|" | \
perl -ne 'if (/^(MD5Sum|SHA1):/i) { $f=1; next }
if ($f) {
unless (/^ /) { print; $f=0 }
} else { print }' > dists/$CODENAME/non-US/Release
else
echo -n "ERROR: Release file ($NONUS/dists/$CODENAME/non-US/Release)"
echo " is missing !"
exit 1
echo "ERROR: Release file ($MIRROR/dists/$CODENAME/Release) is missing !"
exit 1
fi
if [ -n "$NONUS" ] ; then
if [ -e "$NONUS/dists/$CODENAME/non-US/Release" ] ; then
# Strip the MD5Sum and SHA1 fields
# Update some other information as well
sed -e "/^ /d" -e "s/^Architectures: .*$/Architectures: $ARCH/" \
$NONUS/dists/$CODENAME/non-US/Release | \
sed -e "s|^Components: .*$|Components: $SECTIONSNONUS|" | \
perl -ne 'if (/^(MD5Sum|SHA1):/i) { $f=1; next }
if ($f) {
unless (/^ /) { print; $f=0 }
} else { print }' > dists/$CODENAME/non-US/Release
else
echo -n "ERROR: Release file ($NONUS/dists/$CODENAME/non-US/Release)"
echo " is missing !"
exit 1
fi
fi
# Copying release files
for SECT in $SECTIONS; do
# Install the release files
if [ -e "$MIRROR/dists/$CODENAME/$SECT/binary-$ARCH/Release" ]; then
cp $MIRROR/dists/$CODENAME/$SECT/binary-$ARCH/Release \
dists/$CODENAME/$SECT/binary-$ARCH/
fi
if [ -n "$NONUS" -a -e "$NONUS/dists/$CODENAME/non-US/$SECT/binary-$ARCH/Release" ]; then
cp $NONUS/dists/$CODENAME/non-US/$SECT/binary-$ARCH/Release \
dists/$CODENAME/non-US/$SECT/binary-$ARCH/
fi
for SECT in $SECTIONS
do
# Install the release files
if [ -e "$MIRROR/dists/$CODENAME/$SECT/binary-$ARCH/Release" ] ; then
cp $MIRROR/dists/$CODENAME/$SECT/binary-$ARCH/Release \
dists/$CODENAME/$SECT/binary-$ARCH/
fi
if [ -n "$NONUS" -a -e "$NONUS/dists/$CODENAME/non-US/$SECT/binary-$ARCH/Release" ] ; then
cp $NONUS/dists/$CODENAME/non-US/$SECT/binary-$ARCH/Release \
dists/$CODENAME/non-US/$SECT/binary-$ARCH/
fi
done
if [ -n "$LOCALDEBS" -a -n "$LOCAL" ]; then
if [ -e $LOCALDEBS/dists/$CODENAME/local/binary-$ARCH/Release ]; then
cp $LOCALDEBS/dists/$CODENAME/local/binary-$ARCH/Release \
dists/$CODENAME/local/binary-$ARCH/
fi
fi
if [ -e "$MIRROR/dists/$DI_CODENAME/main/debian-installer/binary-$ARCH/Release" ]; then
cp $MIRROR/dists/$DI_CODENAME/main/debian-installer/binary-$ARCH/Release \
dists/$CODENAME/main/debian-installer/binary-$ARCH/
fi
# Generating Packages files
apt-ftparchive generate $PREFIX.generate-binary
if [ -n "$NONUS" ]; then
apt-ftparchive generate $PREFIX.generate-binary-non-US
fi
if [ -e "$MIRROR/dists/$DI_CODENAME/main/debian-installer" ]; then
apt-ftparchive generate $PREFIX.generate-binary-debian-installer
fi
fi ## END "$1" = "install"
# Prepare the files needed to generate the Packages files
if [ "$1" != "scan" ]; then exit 0; fi
#
# Create the file lists and the config files for apt-ftparchive generate
#
# We have to scan all possible dists where packages can be
DISTS=""
DISTSNONUS=""
for i in `cd dists; echo *; cd ..`; do
if [ ! -L "dists/$i" -a -d "dists/$i" ]; then
if [ -d "dists/$i/main/binary-$ARCH" -o \
-d "dists/$i/non-free/binary-$ARCH" -o \
-d "dists/$i/contrib/binary-$ARCH" ]; then
DISTS="$DISTS $i"
if [ -n "$LOCALDEBS" -a -n "$LOCAL" ] ; then
if [ -e $LOCALDEBS/dists/$CODENAME/local/binary-$ARCH/Release ] ; then
cp $LOCALDEBS/dists/$CODENAME/local/binary-$ARCH/Release \
dists/$CODENAME/local/binary-$ARCH/
fi
if [ -n "$SECURITY" ]; then
DISTS="$DISTS $i/updates"
fi
if [ -d "dists/$i/non-US/main/binary-$ARCH" -o \
-d "dists/$i/non-US/non-free/binary-$ARCH" -o \
-d "dists/$i/non-US/contrib/binary-$ARCH" ]; then
DISTSNONUS="$DISTSNONUS $i"
fi
fi
done
# Creating the file lists
for SECT in $SECTIONS; do
touch $PREFIX.filelist_$SECT
if [ -d "pool/$SECT" ]; then
find pool/$SECT >>$PREFIX.filelist_$SECT
fi
if [ -d "pool/unstable/$SECT" ]; then
find pool/unstable/$SECT >>$PREFIX.filelist_$SECT
fi
if [ -n "$SECURITY" -a -d "pool/updates/$SECT" ]; then
find pool/updates/$SECT >>$PREFIX.filelist_$SECT
fi
for DIST in $DISTS; do
if [ -d "dists/$DIST/$SECT/binary-$ARCH" ]; then
find dists/$DIST/$SECT/binary-$ARCH >>$PREFIX.filelist_$SECT
fi
done
if [ -n "$NONUS" ]; then
touch $PREFIX.filelist_non-US_$SECT
if [ -d "pool/non-US/$SECT" ]; then
find pool/non-US/$SECT >>$PREFIX.filelist_non-US_$SECT
fi
for DIST in $DISTSNONUS; do
if [ -d "dists/$DIST/non-US/$SECT/binary-$ARCH" ]; then
find dists/$DIST/non-US/$SECT/binary-$ARCH \
>>$PREFIX.filelist_non-US_$SECT
fi
done
fi
done
# Creating the config files
# SAM 2005-11-01:
# Nasty hack - we may need to look at extra override files from sid.
# See if the extra override exists; if not redirect.
EXTRA_CODENAME=$CODENAME
if [ ! -e $BDIR/indices/override.$EXTRA_CODENAME.extra.main.gz ] ; then
EXTRA_CODENAME=sid
fi
cat >$PREFIX.generate-binary <<EOF
Dir::ArchiveDir "$2";
Dir::OverrideDir "$BDIR/indices";
Dir::CacheDir "$APTTMP/$CODENAME-$ARCH/apt-ftparchive-db";
TreeDefault::Contents " ";
Tree "dists/$CODENAME" {
FileList "$PREFIX.filelist_\$(SECTION)";
Sections "$SECTIONS";
Architectures "$ARCH";
BinOverride "override.$CODENAME.\$(SECTION)";
ExtraOverride "override.$EXTRA_CODENAME.extra.\$(SECTION)";
SrcOverride "override.$CODENAME.\$(SECTION).src";
}
EOF
cat >$PREFIX.generate-binary-non-US <<EOF
Dir::ArchiveDir "$2";
Dir::OverrideDir "$BDIR/indices-non-US";
Dir::CacheDir "$APTTMP/$CODENAME-$ARCH/apt-ftparchive-db";
TreeDefault::Contents " ";
Tree "dists/$CODENAME/non-US" {
FileList "$PREFIX.filelist_non-US_\$(SECTION)";
Sections "`echo $SECTIONSNONUS | sed -e 's#non-US/##g'`";
Architectures "$ARCH";
BinOverride "override.$CODENAME.\$(SECTION)";
ExtraOverride "override.$EXTRA_CODENAME.extra.\$(SECTION)";
SrcOverride "override.$CODENAME.\$(SECTION).src";
}
EOF
cat >$PREFIX.generate-binary-debian-installer <<EOF
Dir::ArchiveDir "$2";
Dir::OverrideDir "$BDIR/indices";
Dir::CacheDir "$APTTMP/$CODENAME-$ARCH/apt-ftparchive-db";
TreeDefault::Contents " ";
Tree "dists/$CODENAME/main" {
FileList "$PREFIX.filelist_main";
Sections "debian-installer";
Architectures "$ARCH";
BinOverride "override.$CODENAME.main.\$(SECTION)";
ExtraOverride "override.$EXTRA_CODENAME.extra.main";
SrcOverride "override.$CODENAME.main.src";
Packages::Extensions ".udeb";
}
EOF
if [ -n "$LOCAL" -a -e "${LOCALDEBS:-$MIRROR}/dists/$DI_CODENAME/local/debian-installer" ]; then
cat >>$PREFIX.generate-binary-debian-installer <<EOF
Tree "dists/$CODENAME/local" {
FileList "$PREFIX.filelist_local";
Sections "debian-installer";
Architectures "$ARCH";
BinOverride "override.$CODENAME.local.\$(SECTION)";
ExtraOverride "override.$EXTRA_CODENAME.extra.local";
SrcOverride "override.$CODENAME.local.src";
Packages::Extensions ".udeb";
}
EOF
fi
if [ -e "$MIRROR/dists/$DI_CODENAME/main/debian-installer/binary-$ARCH/Release" ] ; then
cp $MIRROR/dists/$DI_CODENAME/main/debian-installer/binary-$ARCH/Release \
dists/$CODENAME/main/debian-installer/binary-$ARCH/
fi
# Generating Packages files is now done in add_debs; we no longer use
# apt-ftparchive
# Creating the indices directory
if [ ! -d "$BDIR/indices" ]; then
@ -252,17 +106,17 @@ if [ ! -d "$BDIR/indices" ]; then
fi
gunzip -f $BDIR/indices/*.gz
for SECT in $SECTIONS; do
touch $BDIR/indices/override.$CODENAME.$SECT
touch $BDIR/indices/override.$EXTRA_CODENAME.extra.$SECT
touch $BDIR/indices/override.$CODENAME.$SECT.src
touch $BDIR/indices/override.$CODENAME.$SECT
touch $BDIR/indices/override.$EXTRA_CODENAME.extra.$SECT
touch $BDIR/indices/override.$CODENAME.$SECT.src
done
if [ -e "$MIRROR/dists/$DI_CODENAME/main/debian-installer" ]; then
touch $BDIR/indices/override.$CODENAME.main.debian-installer
touch $BDIR/indices/override.$CODENAME.main.debian-installer
fi
if [ -n "$LOCAL" -a -e "${LOCALDEBS:-$MIRROR}/dists/$DI_CODENAME/local/debian-installer" ]; then
touch $BDIR/indices/override.$CODENAME.local.debian-installer
touch $BDIR/indices/override.$EXTRA_CODENAME.extra.local
touch $BDIR/indices/override.$CODENAME.local.src
touch $BDIR/indices/override.$CODENAME.local.debian-installer
touch $BDIR/indices/override.$EXTRA_CODENAME.extra.local
touch $BDIR/indices/override.$CODENAME.local.src
fi
fi
if [ -n "$NONUS" -a ! -d "$BDIR/indices-non-US" ]; then
@ -270,9 +124,9 @@ if [ -n "$NONUS" -a ! -d "$BDIR/indices-non-US" ]; then
cp $NONUS/indices-non-US/* $BDIR/indices-non-US/
gunzip -f $BDIR/indices-non-US/*.gz
for SECT in `echo $SECTIONSNONUS | sed -e 's#non-US/##g'`; do
touch $BDIR/indices-non-US/override.$CODENAME.$SECT
touch $BDIR/indices-non-US/override.$EXTRA_CODENAME.extra.$SECT
touch $BDIR/indices-non-US/override.$CODENAME.$SECT.src
touch $BDIR/indices-non-US/override.$CODENAME.$SECT
touch $BDIR/indices-non-US/override.$EXTRA_CODENAME.extra.$SECT
touch $BDIR/indices-non-US/override.$CODENAME.$SECT.src
done
fi