From c73d70af8731f556577e11a0f7205e9b864a64c6 Mon Sep 17 00:00:00 2001 From: Steve McIntyre Date: Sun, 9 Feb 2014 15:02:06 +0000 Subject: [PATCH] Code changes to work without local mirror - call grab_file before we use each file that we expect from the mirror --- Makefile | 13 ++++++---- tools/add-bin-doc | 4 ++++ tools/apt-selection | 5 ++++ tools/generate_di_list | 12 ++++++---- tools/grab_file | 2 +- tools/grab_md5 | 36 ++++++++++++++++++++-------- tools/make_disc_trees.pl | 51 +++++++++++++++++++++++----------------- tools/start_new_disc | 4 ++++ 8 files changed, 86 insertions(+), 41 deletions(-) diff --git a/Makefile b/Makefile index 3e04c9e7..56eb0c58 100755 --- a/Makefile +++ b/Makefile @@ -167,10 +167,12 @@ $(BDIR)/DATE: $(Q)date -u '+%Y%m%dT%H%M%SZ' > $(BDIR)/DATE-zulu ifdef MIRROR -LATEST_DB := $(MIRROR)/$(shell $(which_deb) $(MIRROR) $(CODENAME) debootstrap) -$(DB_DIR): $(LATEST_DB) +LATEST_DB := $(shell $(which_deb) $(MIRROR) $(CODENAME) debootstrap) +$(MIRROR)/$(LATEST_DB): + $(Q)grab_file $(LATEST_DB) +$(DB_DIR): $(MIRROR)/$(LATEST_DB) @rm -rf $(DB_DIR) - $(Q)dpkg -x $(LATEST_DB) $(DB_DIR) + $(Q)dpkg -x $(MIRROR)/$(LATEST_DB) $(DB_DIR) $(Q)if [ ! -e $(DEBOOTSTRAP_DIR)/scripts/$(CODENAME) ] ; then \ ln -sf sid $(DEBOOTSTRAP_DIR)/scripts/$(CODENAME) ; \ fi @@ -225,6 +227,7 @@ $(ADIR)/status: if [ $$ARCH = "source" -o "$(INSTALLER_CD)" = "1" -o "$(INSTALLER_CD)" = "2" -o "$(INSTALLER_CD)" = "C" ];then \ :> $(ADIR)/$(CODENAME)-$$ARCH/status ; \ else \ + grab_file $(MIRROR)/dists/$(CODENAME)/main/binary-$$ARCH/Packages.gz; \ zcat $(MIRROR)/dists/$(CODENAME)/main/binary-$$ARCH/Packages.gz | \ perl -000 -ne 's/^(Package: .*)$$/$$1\nStatus: install ok installed/m; print if (/^Priority: (required|important|standard)/m or /^Section: base/m);' \ >> $(ADIR)/$(CODENAME)-$$ARCH/status ; \ @@ -235,7 +238,9 @@ $(ADIR)/status: # Set up keyring so apt doesn't complain @echo "Setting up archive-keyring" $(Q)mkdir -p $(TDIR)/archive-keyring - $(Q)dpkg -x $(MIRROR)/$(shell $(which_deb) $(MIRROR) $(CODENAME) $(ARCHIVE_KEYRING_PACKAGE)) $(TDIR)/archive-keyring + $(Q)AKDEB=$(shell $(which_deb) $(MIRROR) $(CODENAME) $(ARCHIVE_KEYRING_PACKAGE)); \ + grab_file $$AKDEB; \ + dpkg -x $(MIRROR)/$$AKDEB $(TDIR)/archive-keyring $(Q)for ARCH in $(ARCHES); do \ mkdir -p $(ADIR)/$(CODENAME)-$$ARCH/apt/trusted.gpg.d; \ ln -s $(TDIR)/archive-keyring/$(ARCHIVE_KEYRING_FILE) $(ADIR)/$(CODENAME)-$$ARCH/apt/trusted.gpg.d; \ diff --git a/tools/add-bin-doc b/tools/add-bin-doc index 79d8bb5d..312ce73b 100755 --- a/tools/add-bin-doc +++ b/tools/add-bin-doc @@ -27,6 +27,10 @@ if [ "$FORCE_FIRMWARE"x = ""x ] ; then fi dir=$BDIR/CD$DISK +grab_file indices/md5sums.gz +if [ -e $MIRROR/indices/md5sums.gz ] ; then + zcat $MIRROR/indices/md5sums.gz | awk '/ README/ {print $2}' | xargs grab_file +fi for file in $MIRROR/README*; do if [ -e $file ]; then cp -pdf $file $dir/ diff --git a/tools/apt-selection b/tools/apt-selection index 785e743f..a162602d 100755 --- a/tools/apt-selection +++ b/tools/apt-selection @@ -101,10 +101,15 @@ if [ "$1" = "update" ] || [ "$1" = "check" ]; then for section in $sections; do if [ $ARCH = "source" ] ; then grab_file dists/$CODENAME/$section/source/Sources.gz + grab_file dists/$CODENAME/$section/source/Release else grab_file dists/$CODENAME/$section/binary-$ARCH/Packages.gz + grab_file dists/$CODENAME/$section/binary-$ARCH/Release + grab_file dists/$DI_CODENAME/$section/debian-installer/binary-$ARCH/Packages.gz + grab_file dists/$DI_CODENAME/$section/debian-installer/binary-$ARCH/Release fi done + grab_file dists/$CODENAME/Release dists/$CODENAME/Release.gpg apt-get $options $@ exit $? elif [ "$1" = "cache" ]; then diff --git a/tools/generate_di_list b/tools/generate_di_list index 0ba510b1..8e970327 100755 --- a/tools/generate_di_list +++ b/tools/generate_di_list @@ -39,7 +39,8 @@ print OUT << "EOF"; EOF my @common_excludes = read_exclude("exclude-udebs"); -my $mirror_path = "$ENV{MIRROR}/dists/$ENV{DI_CODENAME}"; +my $mirror; +my $mirror_path = "dists/$ENV{DI_CODENAME}"; my @components = qw(main); push @components, 'contrib' if $ENV{CONTRIB}; push @components, 'non-free' if $ENV{NONFREE}; @@ -50,10 +51,13 @@ foreach my $arch (@ARCHES) { my $output = ''; for my $component ( @components ) { my $packagefile="$mirror_path/$component/debian-installer/binary-$arch/Packages.gz"; + $mirror = "$ENV{MIRROR}"; if ( $component eq 'local' and $ENV{LOCALDEBS} ) { - $packagefile="$ENV{LOCALDEBS}/dists/$ENV{DI_CODENAME}/local/debian-installer/binary-$arch/Packages.gz"; + $packagefile="dists/$ENV{DI_CODENAME}/local/debian-installer/binary-$arch/Packages.gz"; + $mirror = "$ENV{LOCALDEBS}"; } - if (! -f $packagefile) { + system("grab_file", "$packagefile"); + if (! -f "$mirror/$packagefile") { print "Missing package file for $arch/$component.\n"; next; } @@ -62,7 +66,7 @@ foreach my $arch (@ARCHES) { push @exclude, read_exclude("exclude-udebs-$arch") if -e exclude_path("exclude-udebs-$arch"); - foreach my $udeb (map { chomp; $_ } `zcat $packagefile | awk '/^Package:/ {print \$2}'`) { + foreach my $udeb (map { chomp; $_ } `zcat $mirror/$packagefile | awk '/^Package:/ {print \$2}'`) { $output .= "$udeb\n" unless grep { $udeb =~ /^${_}$/ } @exclude; } } diff --git a/tools/grab_file b/tools/grab_file index 6a7cf634..29ea5235 100755 --- a/tools/grab_file +++ b/tools/grab_file @@ -20,7 +20,7 @@ if [ "$MIRROR"x = ""x ] ; then fi for WANTED in $@; do - + WANTED=$(echo $WANTED | sed "s,^$MIRROR/,,") if [ ! -e "$MIRROR/$WANTED" ] ; then DIR=$(dirname "$WANTED") mkdir -p "$MIRROR/$DIR" diff --git a/tools/grab_md5 b/tools/grab_md5 index af18ae86..11bce6b5 100755 --- a/tools/grab_md5 +++ b/tools/grab_md5 @@ -16,21 +16,31 @@ CODENAME=$3 DI_CODENAME=$4 OUT=$5 +SECTIONS="main" + +if [ "$CONTRIB"x = "1"x ] ; then + SECTIONS="$SECTIONS contrib" +fi +if [ "$NONFREE"x = "1"x ] || [ "$EXTRANONFREE"x = "1"x ] ; then + SECTIONS="$SECTIONS non-free" +fi +if [ "$LOCAL"x = "1"x ] ; then + SECTIONS="$SECTIONS local" +fi + for ARCH in $ARCHES do - LOCATIONS="$MIRROR/dists/$CODENAME/ $MIRROR/dists/$DI_CODENAME/" + LOCATIONS="$MIRROR/dists/$CODENAME $MIRROR/dists/$DI_CODENAME" echo "Looking in $LOCATIONS" - for LOCATION in $LOCATIONS; do - if [ ! -d $LOCATION ]; then - echo "Error: $LOCATION is not a directory" - exit 1 - fi - done - case $ARCH in source) - FILES=`find $LOCATIONS -name Sources.gz` + FILES="" + for SECTION in $SECTIONS; do + for LOCATION in $LOCATIONS; do + FILES="$FILES $LOCATION/$SECTION/Sources.gz" + done + done echo "Using MD5 sums from Sources files:" echo $FILES zcat -f $FILES | MIRROR=$MIRROR perl -ne ' @@ -59,7 +69,13 @@ do }' | sort | uniq >> $OUT ;; *) - FILES=`find $LOCATIONS -name Packages.gz | grep binary-$ARCH` + FILES="" + for SECTION in $SECTIONS; do + for LOCATION in $LOCATIONS; do + FILES="$FILES $LOCATION/$SECTION/binary-$ARCH/Packages.gz" + FILES="$FILES $LOCATION/$SECTION/debian-installer/binary-$ARCH/Packages.gz" + done + done echo "Using MD5 sums from Packages files:" echo $FILES NUM_FILES=`echo $FILES | wc -w` diff --git a/tools/make_disc_trees.pl b/tools/make_disc_trees.pl index 6d2aa5bf..a0978108 100755 --- a/tools/make_disc_trees.pl +++ b/tools/make_disc_trees.pl @@ -387,7 +387,7 @@ sub load_packages_cache { sub load_descriptions { my $suite = shift; my $lang; - my $dh; + my $dh; my ($p); my $num_total = 0; my $num_files = 0; @@ -397,31 +397,37 @@ sub load_descriptions { } my @files; - if (-d $dir) { + system("grab_file", "$dir/Index"); + if (-e "$dir/Index") { print "Reading in translated package descriptions for $suite:\n"; print LOG "Reading in translated package descriptions for $suite:\n"; - opendir($dh, $dir) || die "can't opendir $dir: $!\n"; - @files = readdir($dh); + open (INDEX, "< $dir/Index") or die "can't open Index file $dir/Index: $!\n"; + while (my $line = ) { + chomp $line; + if ($line =~ /^ [[:xdigit:]]{32}\s+\d+\s+(Translation-(.*).bz2)/) { + push (@files, $1); + } + } + close(INDEX); $/ = ''; # Browse by paragraph foreach my $file (@files) { - if ($file =~ /Translation-(.*).bz2/) { - my $num_descs = 0; - $lang = $1; - open(BZ, "bzip2 -cd $dir/$file |") || - die "can't open description file $dir/$file for reading: $!\n"; - $num_files++; - print LOG " Parsing $file\n"; - while (defined($_ = )) { - m/^Package: (\S+)/m and $p = $1; - $descriptions{"$lang"}{$p}{"data"} = $_; - $descriptions{"$lang"}{$p}{"used"} = 0; - $num_descs++; - $num_total++; - } - close(BZ); - print LOG " $num_descs descriptions\n"; - } - } + system("grab_file", "$dir/$file"); + my $num_descs = 0; + $lang = $1; + open(BZ, "bzip2 -cd $dir/$file |") || + die "can't open description file $dir/$file for reading: $!\n"; + $num_files++; + print LOG " Parsing $file\n"; + while (defined($_ = )) { + m/^Package: (\S+)/m and $p = $1; + $descriptions{"$lang"}{$p}{"data"} = $_; + $descriptions{"$lang"}{$p}{"used"} = 0; + $num_descs++; + $num_total++; + } + close(BZ); + print LOG " $num_descs descriptions\n"; + } $/ = $old_split; # Browse by line again print " Done: read $num_total entries for $num_files languages\n"; print LOG " Done: read $num_total entries for $num_files languages\n"; @@ -1360,6 +1366,7 @@ sub add_packages { foreach my $file (@files) { # And put the file in the CD tree (with a (hard) link) + system("grab_file", "$source/$file"); if (! -e "$source/$file") { msg_ap(0, "Can't find $file in the main archive, trying local\n"); if (-e "$localdebs/$file") { diff --git a/tools/start_new_disc b/tools/start_new_disc index 6fff29d6..96433916 100755 --- a/tools/start_new_disc +++ b/tools/start_new_disc @@ -148,6 +148,10 @@ fi if [ $DISKNUM = 1 ] ; then if [ "$OMIT_DOC_TOOLS" != "1" ]; then + grab_file indices/md5sums.gz + if [ -e $MIRROR/indices/md5sums.gz ] ; then + zcat $MIRROR/indices/md5sums.gz | awk '/ doc/ {print $2}' | xargs grab_file + fi if [ ! -e $MIRROR/doc ]; then echo "You need a Debian mirror with the doc directory!" exit 1