From a55bc64a77d0ca6356a8d593745b9a80e59e6ad3 Mon Sep 17 00:00:00 2001 From: Steve McIntyre Date: Tue, 15 Nov 2016 09:06:41 +0000 Subject: [PATCH] Add support for archives without Packages.gz and Sources.gz Try .xz files too --- Makefile | 9 +++--- debian/changelog | 2 ++ tools/catz | 14 +++++++++ tools/generate_di_list | 24 ++++++++++----- tools/generate_firmware_task | 4 ++- tools/grab_md5 | 13 +++++---- tools/jessie/installtools.sh | 4 +-- tools/make-firmware-image | 2 +- tools/stretch/installtools.sh | 4 +-- tools/wheezy/installtools.sh | 4 +-- tools/which_deb | 55 +++++++++++++++++++++-------------- 11 files changed, 87 insertions(+), 48 deletions(-) create mode 100755 tools/catz diff --git a/Makefile b/Makefile index 03fce49c..6688b529 100755 --- a/Makefile +++ b/Makefile @@ -78,6 +78,7 @@ merge_package_lists=$(BASEDIR)/tools/merge_package_lists update_popcon=$(BASEDIR)/tools/update_popcon grab_source_list=$(BASEDIR)/tools/grab_source_list which_deb=$(BASEDIR)/tools/which_deb +catz=$(BASEDIR)/tools/catz BDIR=$(TDIR)/$(CODENAME) TASKDIR=$(BDIR)/tasks @@ -236,13 +237,13 @@ $(ADIR)/status: :> $(ADIR)/$(CODENAME)-backports-$$ARCH/status ; \ fi; \ else \ - zcat $(MIRROR)/dists/$(CODENAME)/main/binary-$$ARCH/Packages.gz | \ + $(catz) $(MIRROR)/dists/$(CODENAME)/main/binary-$$ARCH/Packages.*z* | \ perl -000 -ne 's/^(Package: .*)$$/$$1\nStatus: install ok installed/m; print if (/^Priority: (required|important|standard)/m or /^Section: base/m);' \ - >> $(ADIR)/$(CODENAME)-$$ARCH/status ; \ + |sort | uniq >> $(ADIR)/$(CODENAME)-$$ARCH/status ; \ if [ "$$BACKPORTS"x != ""x ] ; then \ - zcat $(MIRROR)/dists/$(CODENAME)/main/binary-$$ARCH/Packages.gz | \ + $(catz) $(MIRROR)/dists/$(CODENAME)/main/binary-$$ARCH/Packages.*z* | \ perl -000 -ne 's/^(Package: .*)$$/$$1\nStatus: install ok installed/m; print if (/^Priority: (required|important|standard)/m or /^Section: base/m);' \ - >> $(ADIR)/$(CODENAME)-backports-$$ARCH/status ; \ + |sort | uniq >> $(ADIR)/$(CODENAME)-backports-$$ARCH/status ; \ fi; \ fi; \ done; diff --git a/debian/changelog b/debian/changelog index 3da46750..ecc05ce9 100644 --- a/debian/changelog +++ b/debian/changelog @@ -5,6 +5,8 @@ debian-cd (3.1.19) UNRELEASED; urgency=medium * Add workaround for dosfstools issue (#823881) * Add support for adding backports into a build * Remove obsolete dedicated_source script + * Add support for archives without Packages.gz and Sources.gz - try .xz + files too [ Cyril Brulebois ] * Only export WGET_OPTS="--ca-directory /etc/ssl/ca-debian/" when this diff --git a/tools/catz b/tools/catz new file mode 100755 index 00000000..26461435 --- /dev/null +++ b/tools/catz @@ -0,0 +1,14 @@ +#!/bin/sh + +set -e + +for file in $@; do + if [ -f $file ]; then + case $file in + *.gz) + zcat -f $file;; + *.xz) + xzcat -f $file;; + esac + fi +done diff --git a/tools/generate_di_list b/tools/generate_di_list index 6a6ec30c..de6a34b3 100755 --- a/tools/generate_di_list +++ b/tools/generate_di_list @@ -49,23 +49,31 @@ foreach my $arch (@ARCHES) { (my $cpparch = $arch) =~ s/-/_/g; my $output = ''; for my $component ( @components ) { - my $packagefile="$mirror_path/$component/debian-installer/binary-$arch/Packages.gz"; + my $pgz="$mirror_path/$component/debian-installer/binary-$arch/Packages.gz"; + my $pxz="$mirror_path/$component/debian-installer/binary-$arch/Packages.xz"; if ( $component eq 'local' and $ENV{LOCALDEBS} ) { - $packagefile="$ENV{LOCALDEBS}/dists/$ENV{DI_CODENAME}/local/debian-installer/binary-$arch/Packages.gz"; - } - if (! -f $packagefile) { - print "Missing package file for $arch/$component.\n"; - next; + $pgz="$ENV{LOCALDEBS}/dists/$ENV{DI_CODENAME}/local/debian-installer/binary-$arch/Packages.gz"; + $pxz="$ENV{LOCALDEBS}/dists/$ENV{DI_CODENAME}/local/debian-installer/binary-$arch/Packages.xz"; } my @exclude = @common_excludes; push @exclude, read_exclude("exclude-udebs-$arch") if -e exclude_path("exclude-udebs-$arch"); - foreach my $udeb (map { chomp; $_ } `zcat $packagefile | awk '/^Package:/ {print \$2}'`) { - $output .= "$udeb\n" unless grep { $udeb =~ /^${_}$/ } @exclude; + if (-f $pgz) { + foreach my $udeb (map { chomp; $_ } `zcat $pgz | awk '/^Package:/ {print \$2}'`) { + $output .= "$udeb\n" unless grep { $udeb =~ /^${_}$/ } @exclude; + } + } elsif (-f $pxz) { + foreach my $udeb (map { chomp; $_ } `xzcat $pxz | awk '/^Package:/ {print \$2}'`) { + $output .= "$udeb\n" unless grep { $udeb =~ /^${_}$/ } @exclude; + } + } else { + print "Missing package file for $arch/$component.\n"; + next; } } + next unless $output; print OUT "#ifdef ARCH_$cpparch\n"; print OUT $output; diff --git a/tools/generate_firmware_task b/tools/generate_firmware_task index c5d8c350..a160d141 100755 --- a/tools/generate_firmware_task +++ b/tools/generate_firmware_task @@ -35,11 +35,13 @@ if (!defined($codename) || !defined($mirror) || foreach $arch (split(' ', $archlist)) { $pkgfiles = "$pkgfiles $mirror/dists/$codename/*/binary-$arch/Packages.gz"; + $pkgfiles = "$pkgfiles $mirror/dists/$codename/*/binary-$arch/Packages.xz"; } if (defined($localdebs)) { foreach $arch (split(' ', $archlist)) { $pkgfiles = "$pkgfiles $localdebs/dists/$codename/*/binary-$arch/Packages.gz"; + $pkgfiles = "$pkgfiles $localdebs/dists/$codename/*/binary-$arch/Packages.xz"; } } @@ -56,7 +58,7 @@ print OUT " */\n"; print "$0: Checking for firmware packages:\n"; -open (INPKG, "zcat $pkgfiles |") or die "Can't read input package files: $!\n"; +open (INPKG, "\$BASEDIR/tools/catz $pkgfiles |") or die "Can't read input package files: $!\n"; $/ = ''; # Browse by paragraph sub contains_firmware($$) { diff --git a/tools/grab_md5 b/tools/grab_md5 index ac2dd7e3..e56aad76 100755 --- a/tools/grab_md5 +++ b/tools/grab_md5 @@ -1,7 +1,7 @@ #!/bin/sh # # grab_md5 -# (c) 2004 Steve McIntyre +# (c) 2004-2016 Steve McIntyre # # GPL v2 # @@ -33,10 +33,10 @@ do case $ARCH in source) - FILES=`find $LOCATIONS -name Sources.gz` + FILES=`find $LOCATIONS -name Sources.gz -o -name Sources.xz` echo "Using MD5 sums from Sources files:" echo $FILES - zcat -f $FILES | MIRROR=$MIRROR perl -ne ' + $BASEDIR/tools/catz $FILES | MIRROR=$MIRROR perl -ne ' chomp; my %files; my $dir; @@ -62,15 +62,16 @@ do }' | sort | uniq >> $OUT ;; *) - FILES=`find $LOCATIONS -name Packages.gz | grep binary-$ARCH` + FILES=`find $LOCATIONS -name Packages.gz \ + -o -name Packages.xz | grep binary-$ARCH` echo "Using MD5 sums from Packages files:" echo $FILES - NUM_FILES=`echo $FILES | wc -w` + NUM_FILES=`echo $FILES| wc -w` if [ $NUM_FILES -eq 1 ] ; then echo "No files found for arch $ARCH. Abort!" exit 1 fi - zcat -f $FILES | MIRROR=$MIRROR perl -ne ' + catz $FILES | MIRROR=$MIRROR perl -ne ' chomp; my $mirror = $ENV{"MIRROR"}; my $filename; diff --git a/tools/jessie/installtools.sh b/tools/jessie/installtools.sh index d7928ef5..a8ac4b5d 100755 --- a/tools/jessie/installtools.sh +++ b/tools/jessie/installtools.sh @@ -25,8 +25,8 @@ if [ "$OMIT_MANUAL" != 1 ]; then INSTALLDIR=$DIR/$DOCDIR/manual/$ARCH fi - INSTALLGUIDE=$(zcat $MIRROR/dists/$CODENAME/main/binary-$ARCH//Packages.gz | \ - sed -n "s/Filename: \(pool\/main\/i\/installation-guide\/installation-guide-$ARCH.*deb\)$/\1/p") + INSTALLGUIDE=$($BASEDIR/tools/catz $MIRROR/dists/$CODENAME/main/binary-$ARCH//Packages.*z* | \ + sed -n "s/Filename: \(pool\/main\/i\/installation-guide\/installation-guide-$ARCH.*deb\)$/\1/p" | sort | uniq) if [ -f "$MIRROR/$INSTALLGUIDE" ]; then rm -rf $MANTDIR diff --git a/tools/make-firmware-image b/tools/make-firmware-image index 289abbeb..5568cc33 100755 --- a/tools/make-firmware-image +++ b/tools/make-firmware-image @@ -25,7 +25,7 @@ fi rm -rf $TMPDIR/firmware mkdir -p $TMPDIR/firmware -FILES=`zcat ${MIRROR}/dists/${SUITE}/*/binary-*/Packages.gz | \ +FILES=`$BASEDIR/tools/catz ${MIRROR}/dists/${SUITE}/*/binary-*/Packages.*z* | \ grep-dctrl -Pe '.*(firmware|microcode).*' -s Filename -n | sort -u` for FILE in $FILES; do diff --git a/tools/stretch/installtools.sh b/tools/stretch/installtools.sh index d7928ef5..a8ac4b5d 100755 --- a/tools/stretch/installtools.sh +++ b/tools/stretch/installtools.sh @@ -25,8 +25,8 @@ if [ "$OMIT_MANUAL" != 1 ]; then INSTALLDIR=$DIR/$DOCDIR/manual/$ARCH fi - INSTALLGUIDE=$(zcat $MIRROR/dists/$CODENAME/main/binary-$ARCH//Packages.gz | \ - sed -n "s/Filename: \(pool\/main\/i\/installation-guide\/installation-guide-$ARCH.*deb\)$/\1/p") + INSTALLGUIDE=$($BASEDIR/tools/catz $MIRROR/dists/$CODENAME/main/binary-$ARCH//Packages.*z* | \ + sed -n "s/Filename: \(pool\/main\/i\/installation-guide\/installation-guide-$ARCH.*deb\)$/\1/p" | sort | uniq) if [ -f "$MIRROR/$INSTALLGUIDE" ]; then rm -rf $MANTDIR diff --git a/tools/wheezy/installtools.sh b/tools/wheezy/installtools.sh index d7928ef5..e1ecb045 100755 --- a/tools/wheezy/installtools.sh +++ b/tools/wheezy/installtools.sh @@ -25,8 +25,8 @@ if [ "$OMIT_MANUAL" != 1 ]; then INSTALLDIR=$DIR/$DOCDIR/manual/$ARCH fi - INSTALLGUIDE=$(zcat $MIRROR/dists/$CODENAME/main/binary-$ARCH//Packages.gz | \ - sed -n "s/Filename: \(pool\/main\/i\/installation-guide\/installation-guide-$ARCH.*deb\)$/\1/p") + INSTALLGUIDE=$($BASEDIR/tools/catz $MIRROR/dists/$CODENAME/main/binary-$ARCH/Packages.*z* | \ + sed -n "s/Filename: \(pool\/main\/i\/installation-guide\/installation-guide-$ARCH.*deb\)$/\1/p" | sort | uniq) if [ -f "$MIRROR/$INSTALLGUIDE" ]; then rm -rf $MANTDIR diff --git a/tools/which_deb b/tools/which_deb index 3f043928..d5a5a94f 100755 --- a/tools/which_deb +++ b/tools/which_deb @@ -74,20 +74,27 @@ sub grab_bin_info { if ( $component eq 'local' and $ENV{LOCALDEBS} ) { $pgz = "$ENV{LOCALDEBS}/dists/$codename/local/binary-$arch/Packages.gz"; } + my $pxz = "$pth/$component/binary-$arch/Packages.xz"; + if ( $component eq 'local' and $ENV{LOCALDEBS} ) { + $pgz = "$ENV{LOCALDEBS}/dists/$codename/local/binary-$arch/Packages.xz"; + } if (-e $pgz) { open(PFILE, "zcat $pgz |") or die "Failed to read Packages file $pgz"; - - while (defined($match = )) { - if (($match =~ /^Package: \Q$pkgname\E$/m)) { - $result = $match; - close PFILE; - return $result; - } - } - # Fell through - close PFILE; - } + } elsif (-e $pxz) { + open(PFILE, "xzcat $pxz |") or + die "Failed to read Packages file $pxz"; + } else { + die "Can't find a working Packages file, tried $pgz and $pxz\n"; + } + while (defined($match = )) { + if (($match =~ /^Package: \Q$pkgname\E$/m)) { + $result = $match; + close PFILE; + return $result; + } + } + close PFILE; } return ""; } @@ -103,21 +110,25 @@ sub grab_src_info { for my $component ( @components ) { my $pgz = "$pth/$component/source/Sources.gz"; + my $pxz = "$pth/$component/source/Sources.xz"; if (-e $pgz) { open(PFILE, "zcat $pgz |") or die "Failed to read Sources file $pgz"; - - while (defined($match = )) { - if (($match =~ /^Package: \Q$pkgname\E$/m)) { - $result = $match; - close PFILE; - return $result; - } - } - # Fell through - close PFILE; - } + } elsif (-e $pxz) { + open(PFILE, "xzcat $pxz |") or + die "Failed to read Sources file $pxz"; + } else { + die "Can't find a working Sources file, tried $pgz and $pxz\n"; + } + while (defined($match = )) { + if (($match =~ /^Package: \Q$pkgname\E$/m)) { + $result = $match; + close PFILE; + return $result; + } + } + close PFILE; } return ""; }