Code changes to work without local mirror - call grab_file before we

use each file that we expect from the mirror

Makefile

@@ -167,10 +167,12 @@ $(BDIR)/DATE:
 	$(Q)date -u '+%Y%m%dT%H%M%SZ' > $(BDIR)/DATE-zulu
 
 ifdef MIRROR
-LATEST_DB := $(MIRROR)/$(shell $(which_deb) $(MIRROR) $(CODENAME) debootstrap)
+LATEST_DB := $(shell $(which_deb) $(MIRROR) $(CODENAME) debootstrap)
-$(DB_DIR): $(LATEST_DB)
+$(MIRROR)/$(LATEST_DB):
+	$(Q)grab_file $(LATEST_DB)
+$(DB_DIR): $(MIRROR)/$(LATEST_DB)
 	@rm -rf $(DB_DIR)
-	$(Q)dpkg -x $(LATEST_DB) $(DB_DIR)
+	$(Q)dpkg -x $(MIRROR)/$(LATEST_DB) $(DB_DIR)
 	$(Q)if [ ! -e $(DEBOOTSTRAP_DIR)/scripts/$(CODENAME) ] ; then \
 		ln -sf sid $(DEBOOTSTRAP_DIR)/scripts/$(CODENAME) ; \
 	fi
@@ -225,6 +227,7 @@ $(ADIR)/status:
 		if [ $$ARCH = "source" -o "$(INSTALLER_CD)" = "1" -o "$(INSTALLER_CD)" = "2" -o "$(INSTALLER_CD)" = "C" ];then \
 			:> $(ADIR)/$(CODENAME)-$$ARCH/status ; \
 		else \
+			grab_file $(MIRROR)/dists/$(CODENAME)/main/binary-$$ARCH/Packages.gz; \
 			zcat $(MIRROR)/dists/$(CODENAME)/main/binary-$$ARCH/Packages.gz | \
 			perl -000 -ne 's/^(Package: .*)$$/$$1\nStatus: install ok installed/m; print if (/^Priority: (required|important|standard)/m or /^Section: base/m);' \
 			>> $(ADIR)/$(CODENAME)-$$ARCH/status ; \
@@ -235,7 +238,9 @@ $(ADIR)/status:
 	# Set up keyring so apt doesn't complain
 	@echo "Setting up archive-keyring"
 	$(Q)mkdir -p $(TDIR)/archive-keyring
-	$(Q)dpkg -x $(MIRROR)/$(shell $(which_deb) $(MIRROR) $(CODENAME) $(ARCHIVE_KEYRING_PACKAGE)) $(TDIR)/archive-keyring
+	$(Q)AKDEB=$(shell $(which_deb) $(MIRROR) $(CODENAME) $(ARCHIVE_KEYRING_PACKAGE)); \
+		grab_file $$AKDEB; \
+		dpkg -x $(MIRROR)/$$AKDEB $(TDIR)/archive-keyring
 	$(Q)for ARCH in $(ARCHES); do \
 		mkdir -p $(ADIR)/$(CODENAME)-$$ARCH/apt/trusted.gpg.d; \
 		ln -s $(TDIR)/archive-keyring/$(ARCHIVE_KEYRING_FILE) $(ADIR)/$(CODENAME)-$$ARCH/apt/trusted.gpg.d; \

tools/add-bin-doc

@@ -27,6 +27,10 @@ if [ "$FORCE_FIRMWARE"x = ""x ] ; then
 fi
 
 dir=$BDIR/CD$DISK
+grab_file indices/md5sums.gz
+if [ -e $MIRROR/indices/md5sums.gz ] ; then
+    zcat $MIRROR/indices/md5sums.gz | awk '/  README/ {print $2}' | xargs grab_file
+fi
 for file in $MIRROR/README*; do
 	if [ -e $file ]; then
 		cp -pdf $file $dir/ 

tools/apt-selection

@@ -101,10 +101,15 @@ if [ "$1" = "update" ] || [ "$1" = "check" ]; then
 	for section in $sections; do
 		if [ $ARCH = "source" ] ; then
 			grab_file dists/$CODENAME/$section/source/Sources.gz
+			grab_file dists/$CODENAME/$section/source/Release
 		else
 			grab_file dists/$CODENAME/$section/binary-$ARCH/Packages.gz
+			grab_file dists/$CODENAME/$section/binary-$ARCH/Release
+			grab_file dists/$DI_CODENAME/$section/debian-installer/binary-$ARCH/Packages.gz
+			grab_file dists/$DI_CODENAME/$section/debian-installer/binary-$ARCH/Release
 		fi
 	done
+	grab_file dists/$CODENAME/Release dists/$CODENAME/Release.gpg
 	apt-get $options $@
 	exit $?
 elif [ "$1" = "cache" ]; then

tools/generate_di_list

@@ -39,7 +39,8 @@ print OUT << "EOF";
 EOF
 
 my @common_excludes = read_exclude("exclude-udebs");
-my $mirror_path = "$ENV{MIRROR}/dists/$ENV{DI_CODENAME}";
+my $mirror;
+my $mirror_path = "dists/$ENV{DI_CODENAME}";
 my @components = qw(main);
 push @components, 'contrib' if $ENV{CONTRIB};
 push @components, 'non-free' if $ENV{NONFREE};
@@ -50,10 +51,13 @@ foreach my $arch (@ARCHES) {
     my $output = '';
     for my $component ( @components ) {
         my $packagefile="$mirror_path/$component/debian-installer/binary-$arch/Packages.gz";
+	$mirror = "$ENV{MIRROR}";
         if ( $component eq 'local' and $ENV{LOCALDEBS} ) {
-            $packagefile="$ENV{LOCALDEBS}/dists/$ENV{DI_CODENAME}/local/debian-installer/binary-$arch/Packages.gz";
+            $packagefile="dists/$ENV{DI_CODENAME}/local/debian-installer/binary-$arch/Packages.gz";
+	    $mirror = "$ENV{LOCALDEBS}";
         }
-        if (! -f $packagefile) {
+	system("grab_file", "$packagefile");
+        if (! -f "$mirror/$packagefile") {
             print "Missing package file for $arch/$component.\n";
             next;
         }
@@ -62,7 +66,7 @@ foreach my $arch (@ARCHES) {
         push @exclude, read_exclude("exclude-udebs-$arch")
             if -e exclude_path("exclude-udebs-$arch");
 
-        foreach my $udeb (map { chomp; $_ } `zcat $packagefile | awk '/^Package:/ {print \$2}'`) {
+        foreach my $udeb (map { chomp; $_ } `zcat $mirror/$packagefile | awk '/^Package:/ {print \$2}'`) {
             $output .= "$udeb\n" unless grep { $udeb =~ /^${_}$/ } @exclude;
         }
     }

tools/grab_file

@@ -20,7 +20,7 @@ if [ "$MIRROR"x = ""x ] ; then
 fi
 
 for WANTED in $@; do
-
+    WANTED=$(echo $WANTED | sed "s,^$MIRROR/,,")
     if [ ! -e "$MIRROR/$WANTED" ] ; then
 	DIR=$(dirname "$WANTED")
 	mkdir -p "$MIRROR/$DIR"

tools/grab_md5

@@ -16,21 +16,31 @@ CODENAME=$3
 DI_CODENAME=$4
 OUT=$5
 
+SECTIONS="main"
+
+if [ "$CONTRIB"x = "1"x ] ; then
+    SECTIONS="$SECTIONS contrib"
+fi
+if [ "$NONFREE"x = "1"x ] || [ "$EXTRANONFREE"x = "1"x ] ; then
+    SECTIONS="$SECTIONS non-free"
+fi
+if [ "$LOCAL"x = "1"x ] ; then
+    SECTIONS="$SECTIONS local"
+fi
+
 for ARCH in $ARCHES
 do
-    LOCATIONS="$MIRROR/dists/$CODENAME/ $MIRROR/dists/$DI_CODENAME/"
+    LOCATIONS="$MIRROR/dists/$CODENAME $MIRROR/dists/$DI_CODENAME"
     echo "Looking in $LOCATIONS"
 
-    for LOCATION in $LOCATIONS; do
-        if [ ! -d $LOCATION ]; then
-            echo "Error: $LOCATION is not a directory"
-            exit 1
-        fi
-    done
-
     case $ARCH in
         source)
-            FILES=`find $LOCATIONS -name Sources.gz`
+	    FILES=""
+	    for SECTION in $SECTIONS; do
+		for LOCATION in $LOCATIONS; do
+		    FILES="$FILES $LOCATION/$SECTION/Sources.gz"
+		done
+	    done
             echo "Using MD5 sums from Sources files:"
             echo $FILES
             zcat -f $FILES | MIRROR=$MIRROR perl -ne '
@@ -59,7 +69,13 @@ do
                 }' | sort | uniq >> $OUT
             ;;
         *)
-            FILES=`find $LOCATIONS -name Packages.gz | grep binary-$ARCH`
+	    FILES=""
+	    for SECTION in $SECTIONS; do
+		for LOCATION in $LOCATIONS; do
+		    FILES="$FILES $LOCATION/$SECTION/binary-$ARCH/Packages.gz"
+		    FILES="$FILES $LOCATION/$SECTION/debian-installer/binary-$ARCH/Packages.gz"
+		done
+	    done
             echo "Using MD5 sums from Packages files:"
             echo $FILES
             NUM_FILES=`echo $FILES | wc -w`

tools/make_disc_trees.pl

@@ -397,14 +397,21 @@ sub load_descriptions {
     }
     my @files;
 
-    if (-d $dir) {
+    system("grab_file", "$dir/Index");
+    if (-e "$dir/Index") {
         print "Reading in translated package descriptions for $suite:\n";
         print LOG "Reading in translated package descriptions for $suite:\n";
-        opendir($dh, $dir) || die "can't opendir $dir: $!\n";
+	open (INDEX, "< $dir/Index") or die "can't open Index file $dir/Index: $!\n";
-        @files = readdir($dh);
+	while (my $line = <INDEX>) {
+	    chomp $line;
+	    if ($line =~ /^ [[:xdigit:]]{32}\s+\d+\s+(Translation-(.*).bz2)/) {
+		push (@files, $1);
+	    }
+	}
+	close(INDEX);
         $/ = ''; # Browse by paragraph
         foreach my $file (@files) {
-            if ($file =~ /Translation-(.*).bz2/) {
+	    system("grab_file", "$dir/$file");
 	    my $num_descs = 0;
 	    $lang = $1;
 	    open(BZ, "bzip2 -cd $dir/$file |") ||
@@ -421,7 +428,6 @@ sub load_descriptions {
 	    close(BZ);
 	    print LOG "    $num_descs descriptions\n";
 	}
-        }
         $/ = $old_split; # Browse by line again
         print "  Done: read $num_total entries for $num_files languages\n";
         print LOG "  Done: read $num_total entries for $num_files languages\n";
@@ -1360,6 +1366,7 @@ sub add_packages {
             foreach my $file (@files) {
 
                 # And put the file in the CD tree (with a (hard) link)
+		system("grab_file", "$source/$file");
                 if (! -e "$source/$file") {
                     msg_ap(0, "Can't find $file in the main archive, trying local\n");
                     if (-e "$localdebs/$file") {

tools/start_new_disc

@@ -148,6 +148,10 @@ fi
 
 if [ $DISKNUM = 1 ] ; then
     if [ "$OMIT_DOC_TOOLS" != "1" ]; then
+	grab_file indices/md5sums.gz
+	if [ -e $MIRROR/indices/md5sums.gz ] ; then
+	    zcat $MIRROR/indices/md5sums.gz | awk '/  doc/ {print $2}' | xargs grab_file
+	fi
         if [ ! -e $MIRROR/doc ]; then
             echo "You need a Debian mirror with the doc directory!"
             exit 1