Added code/docs for automatic generation of fallback directories during jigdo generation

CONF.sh

@@ -26,6 +26,9 @@ unset NOSUGGESTS        || true
 unset DOJIGDO           || true
 unset JIGDOCMD          || true
 unset JIGDOTEMPLATEURL  || true
+unset JIGDOFALLBACKURLS || true
+unset JIGDOINCLUDEURLS  || true
+unset JIGDOSCRIPT       || true
 unset DEFBINSIZE        || true
 unset DEFSRCSIZE        || true
 unset FASTSUMS          || true
@@ -179,18 +182,47 @@ export DEFSRCSIZE=635
 # Note: building the cache takes hours, so keep it around for the next run
 #export JIGDOCMD="/usr/local/bin/jigdo-file --cache=$HOME/jigdo-cache.db"
 #
-# HTTP/FTP URL for directory where you intend to make the templates available.
+# HTTP/FTP URL for directory where you intend to make the templates
-# %ARCH%, if present, will be replaced by $ARCH (or "source"). This only goes
+# available. You should not need to change this; the default value ""
-# in the .jigdo files, which you can edit easily if you want.
+# means "template in same dir as the .jigdo file", which is usually
-# No trailing slash.
+# correct. If it is non-empty, it needs a trailing slash. "%ARCH%"
-#export JIGDOTEMPLATEURL="http://this-guy-didnt-configure-debiancd-correctly.com/debian-cd/templates/3.0BETA/%ARCH%"
+# will be substituted by the current architecture.
+#export JIGDOTEMPLATEURL=""
+#
+# Name of a directory on disc to create data for a fallback server in. 
+# Should later be made available by you at the URL given in
+# JIGDOFALLBACKURLS. In the directory, two subdirs named "Debian" and
+# "Non-US" will be created, and filled with hard links to the actual
+# files in your FTP archive. Because of the hard links, the dir must
+# be on the same partition as the FTP archive! If unset, no fallback
+# data is created, which may cause problems - see README.
+#export JIGDOFALLBACKPATH="$(OUT)/snapshot/"
+#
+# Space-separated list of label->URL mappings for "jigdo fallback
+# server(s)" to add to .jigdo file. If unset, no fallback URL is
+# added, which may cause problems - see README.
+#export JIGDOFALLBACKURLS="Debian=http://myserver/snapshot/Debian/ Non-US=http://myserver/snapshot/Non-US/"
+#
+# Space-separated list of "include URLs" to add to the .jigdo file. 
+# The included files are used to provide an up-to-date list of Debian
+# mirrors to the jigdo _GUI_application_ (_jigdo-lite_ doesn't support
+# "[Include ...]").
+export JIGDOINCLUDEURLS="http://cdimage.debian.org/debian-cd/debian-servers.jigdo"
+#
+# $JIGDOTEMPLATEURL and $JIGDOINCLUDEURLS are passed to
+# "tools/jigdo_header", which is used by default to generate the
+# [Image] and [Servers] sections of the .jigdo file. You can provide
+# your own script if you need the .jigdo file to contain different
+# data.
+#export JIGDOSCRIPT="myscript"
 
 # If set, use the md5sums from the main archive, rather than calculating
 # them locally
 #export FASTSUMS=1
 
-# a couple of things used by publish_cds, so it can tweak the jigdo files,
+# A couple of things used only by publish_cds, so it can tweak the
-# and knows where to put the results
+# jigdo files, and knows where to put the results.
+# You need to run publish_cds manually, it is not run by the Makefile.
 export PUBLISH_URL="http://cdimage.debian.org/jigdo-area"
 export PUBLISH_NONUS_URL="http://non-US.cdimage.debian.org/jigdo-area"
 export PUBLISH_PATH="/home/jigdo-area/"

Makefile

@@ -24,17 +24,17 @@ ifndef CAPCODENAME
 CAPCODENAME:=$(shell perl -e "print ucfirst("$(CODENAME)")")
 endif
 ifndef BINDISKINFO
-BINDISKINFO="Debian GNU/Linux $(DEBVERSION) \"$(CAPCODENAME)\" - $(OFFICIAL) $(ARCH) Binary-$$num ($$DATE)"
+export BINDISKINFO="Debian GNU/Linux $(DEBVERSION) \"$(CAPCODENAME)\" - $(OFFICIAL) $(ARCH) Binary-$$num ($$DATE)"
 endif
 ifndef SRCDISKINFO
-SRCDISKINFO="Debian GNU/Linux $(DEBVERSION) \"$(CAPCODENAME)\" - $(OFFICIAL) Source-$$num ($$DATE)"
+export SRCDISKINFO="Debian GNU/Linux $(DEBVERSION) \"$(CAPCODENAME)\" - $(OFFICIAL) Source-$$num ($$DATE)"
 endif
 # ND=No-Date versions for README
 ifndef BINDISKINFOND
-BINDISKINFOND="Debian GNU/Linux $(DEBVERSION) \"$(CAPCODENAME)\" - $(OFFICIAL) $(ARCH) Binary-$$num"
+export BINDISKINFOND="Debian GNU/Linux $(DEBVERSION) \"$(CAPCODENAME)\" - $(OFFICIAL) $(ARCH) Binary-$$num"
 endif
 ifndef SRCDISKINFOND
-SRCDISKINFOND="Debian GNU/Linux $(DEBVERSION) \"$(CAPCODENAME)\" - $(OFFICIAL) Source-$$num"
+export SRCDISKINFOND="Debian GNU/Linux $(DEBVERSION) \"$(CAPCODENAME)\" - $(OFFICIAL) Source-$$num"
 endif
 ifndef BINVOLID
 ifeq ($(ARCH),powerpc)
@@ -66,6 +66,9 @@ endif
 ifndef DOJIGDO
 export DOJIGDO=0
 endif
+ifndef JIGDOSCRIPT
+JIGDOSCRIPT=$(BASEDIR)/tools/jigdo_header
+endif
 
 ifndef UDEB_INCLUDE
 # Netinst/businesscard CD should have udeb_include file by default
@@ -165,9 +168,6 @@ ifneq "$(DOJIGDO)" "0"
 ifndef JIGDOCMD
 	@echo JIGDOCMD undefined -- set up CONF.sh; false
 endif
-ifndef JIGDOTEMPLATEURL
-	@echo JIGDOTEMPLATEURL undefined -- set up CONF.sh; false
-endif
 else
 export JIGDOCMD=false
 endif
@@ -846,18 +846,11 @@ bin-images: ok bin-md5list $(OUT) $(TDIR)/jigdofilelist
 		volid=`cat $(BDIR)/$$n.volid`; \
 		rm -f $(OUT)/$(CODENAME)-$(ARCH)-$$n.raw; \
 		if [ "$(DOJIGDO)" != "0" ]; then \
-			echo "[Image]" \
+			$(JIGDOSCRIPT) \
+				"debian-`echo $(DEBVERSION) | sed -e 's/[. ]//g'`-$(ARCH)-binary-$$n.iso" \
+				"`echo "$(JIGDOTEMPLATEURL)" | sed -e 's|%ARCH%|$(ARCH)|g'`$(CODENAME)-$(ARCH)-$$n.template" \
+				$(BINDISKINFOND) \
 				> $(TDIR)/$(CODENAME)-$(ARCH).jigdo; \
-			echo "Filename=debian-`echo $(DEBVERSION) | sed -e 's/[. ]//g'`-$(ARCH)-binary-$$n.iso" \
-				>> $(TDIR)/$(CODENAME)-$(ARCH).jigdo; \
-			echo "Template=`echo "$(JIGDOTEMPLATEURL)" | sed -e 's|%ARCH%|$(ARCH)|g'`/$(CODENAME)-$(ARCH)-$$n.template" \
-				>> $(TDIR)/$(CODENAME)-$(ARCH).jigdo; \
-			echo "Template-MD5Sum=" \
-				>> $(TDIR)/$(CODENAME)-$(ARCH).jigdo; \
-			echo "ShortInfo='"$(BINDISKINFOND)" CD'" \
-				>> $(TDIR)/$(CODENAME)-$(ARCH).jigdo; \
-			echo "Info='Generated on `date -R`'" \
-				>> $(TDIR)/$(CODENAME)-$(ARCH).jigdo; \
 		fi; \
 		if [ "$(DOJIGDO)" != "2" -o -f $(BASEDIR)/tools/boot/$(CODENAME)/post-boot-$(ARCH) ]; then \
 			$(MKISOFS) $(MKISOFS_OPTS) -V "$$volid" \
@@ -867,30 +860,18 @@ bin-images: ok bin-md5list $(OUT) $(TDIR)/jigdofilelist
 				 $(OUT)/$(CODENAME)-$(ARCH)-$$n.raw; \
 			fi; \
 			if [ "$(DOJIGDO)" != "0" ]; then \
-				$(JIGDOCMD) make-template --force \
+				$(BASEDIR)/tools/jigdo_create "$(OUT)/$(CODENAME)-$(ARCH)-$$n.raw" \
-				  --files-from=$(TDIR)/jigdofilelist \
+				  "$(OUT)/$(CODENAME)-$(ARCH)-$$n.jigdo" \
-				  --image=$(OUT)/$(CODENAME)-$(ARCH)-$$n.raw \
+				  "$(OUT)/$(CODENAME)-$(ARCH)-$$n.template" \
-				  --jigdo=$(OUT)/$(CODENAME)-$(ARCH)-$$n.jigdo \
+				  "$(TDIR)/$(CODENAME)-$(ARCH).jigdo"; \
-				  --template=$(OUT)/$(CODENAME)-$(ARCH)-$$n.template \
-				  --label Non-US="$(NONUS)" \
-				  --label Debian="$(MIRROR)" \
-				  --merge="$(TDIR)/$(CODENAME)-$(ARCH).jigdo" \
-				  --no-servers-section \
-				  --report=noprogress; \
 			fi; \
 		else \
 			$(MKISOFS) $(MKISOFS_OPTS) -V "$$volid" \
 			  $$opts CD$$n \
-			| $(JIGDOCMD) make-template --force \
+			| $(BASEDIR)/tools/jigdo_create "-" \
-				  --files-from=$(TDIR)/jigdofilelist \
+				  "$(OUT)/$(CODENAME)-$(ARCH)-$$n.jigdo" \
-				  --image=- \
+				  "$(OUT)/$(CODENAME)-$(ARCH)-$$n.template" \
-				  --jigdo=$(OUT)/$(CODENAME)-$(ARCH)-$$n.jigdo \
+				  "$(TDIR)/$(CODENAME)-$(ARCH).jigdo"; \
-				  --template=$(OUT)/$(CODENAME)-$(ARCH)-$$n.template \
-				  --label Non-US="$(NONUS)" \
-				  --label Debian="$(MIRROR)" \
-				  --merge="$(TDIR)/$(CODENAME)-$(ARCH).jigdo" \
-				  --no-servers-section \
-				  --report=noprogress; \
 		fi; \
 		if [ "$(DOJIGDO)" = "2" ]; then \
 			rm -f $(OUT)/$(CODENAME)-$(ARCH)-$$n.raw; \
@@ -910,47 +891,28 @@ src-images: ok src-md5list $(OUT) $(TDIR)/jigdofilelist
 		volid=`cat $(SDIR)/$$n.volid`; \
 		rm -f $(OUT)/$(CODENAME)-src-$$n.raw; \
 		if [ "$(DOJIGDO)" != "0" ]; then \
-			echo "[Image]" \
+			$(JIGDOSCRIPT) \
+				"debian-`echo $(DEBVERSION) | sed -e 's/[. ]//g'`-source-$$n.iso" \
+				"`echo "$(JIGDOTEMPLATEURL)" | sed -e 's|%ARCH%|$(ARCH)|g'`$(CODENAME)-src-$$n.template" \
+				$(SRCDISKINFOND) \
 				> $(TDIR)/$(CODENAME)-src.jigdo; \
-			echo "Filename=debian-`echo $(DEBVERSION) | sed -e 's/[. ]//g'`-source-$$n.iso" \
-				>> $(TDIR)/$(CODENAME)-src.jigdo; \
-			echo "Template=`echo "$(JIGDOTEMPLATEURL)" | sed -e 's|%ARCH%|source|g'`/$(CODENAME)-src-$$n.template" \
-				>> $(TDIR)/$(CODENAME)-src.jigdo; \
-			echo "Template-MD5Sum=" \
-				>> $(TDIR)/$(CODENAME)-src.jigdo; \
-			echo "ShortInfo='"$(SRCDISKINFOND)" CD'" \
-				>> $(TDIR)/$(CODENAME)-src.jigdo; \
-			echo "Info='Generated on `date -R`'" \
-				>> $(TDIR)/$(CODENAME)-src.jigdo; \
 		fi; \
 		if [ "$(DOJIGDO)" != "2" ]; then \
 			$(MKISOFS) $(MKISOFS_OPTS) -V "$$volid" \
 			  -o $(OUT)/$(CODENAME)-src-$$n.raw $$opts CD$$n ; \
 			if [ "$(DOJIGDO)" != "0" ]; then \
-				$(JIGDOCMD) make-template --force \
+				$(BASEDIR)/tools/jigdo_create "$(OUT)/$(CODENAME)-src-$$n.raw" \
-				  --files-from=$(TDIR)/jigdofilelist \
+				  "$(OUT)/$(CODENAME)-src-$$n.jigdo" \
-				  --image=$(OUT)/$(CODENAME)-src-$$n.raw \
+				  "$(OUT)/$(CODENAME)-src-$$n.template" \
-				  --jigdo=$(OUT)/$(CODENAME)-src-$$n.jigdo \
+				  "$(TDIR)/$(CODENAME)-src.jigdo"; \
-				  --template=$(OUT)/$(CODENAME)-src-$$n.template \
-				  --label Non-US="$(NONUS)" \
-				  --label Debian="$(MIRROR)" \
-				  --merge="$(TDIR)/$(CODENAME)-src.jigdo" \
-				  --no-servers-section \
-				  --report=noprogress; \
 			fi; \
 		else \
 			$(MKISOFS) $(MKISOFS_OPTS) -V "$$volid" \
 			  $$opts CD$$n \
-			| $(JIGDOCMD) make-template --force \
+			| $(BASEDIR)/tools/jigdo_create "-" \
-				  --files-from=$(TDIR)/jigdofilelist \
+				  "$(OUT)/$(CODENAME)-src-$$n.jigdo" \
-				  --image=- \
+				  "$(OUT)/$(CODENAME)-src-$$n.template" \
-				  --jigdo=$(OUT)/$(CODENAME)-src-$$n.jigdo \
+				  "$(TDIR)/$(CODENAME)-src.jigdo"; \
-				  --template=$(OUT)/$(CODENAME)-src-$$n.template \
-				  --label Non-US="$(NONUS)" \
-				  --label Debian="$(MIRROR)" \
-				  --merge="$(TDIR)/$(CODENAME)-src.jigdo" \
-				  --no-servers-section \
-				  --report=noprogress; \
 		fi; \
 	done
 	rm -f "$(TDIR)/$(CODENAME)-src.jigdo"

README

@@ -317,6 +317,33 @@ no full iso image is available, the MD5sum will be extracted from the
 .template file. A note in the Makefile shows how the original file size
 can be extracted from the .template in a similar way.
 
+About jigdo "fallback servers":
+
+jigdo works by downloading individual packages and other files from a
+normal Debian mirror, and using them to regenerate a CD/DVD image. 
+However, the content of Debian mirrors changes over time, files are
+added and removed. But jigdo must have access to all files needed for
+the image it has to regenerate, even those that have been removed from
+the normal Debian mirrors.
+
+A fallback server contains a backup of the Debian FTP space for the
+moment the .jigdo files were generated. This backup is made available
+under a certain URL which is written to the .jigdo files. jigdo will
+*only* revert to the fallback server after an unsuccessful attempt to
+retrieve a file from the normal user-selected Debian mirror, so the
+bandwidth requirements are modest.
+
+A fallback is even necessary for .jigdo files of the stable release,
+because some files (typically documentation or boot floppies) can
+change at any time.
+
+debian-cd allows you to automatically create a directory on disc which
+is suitable for use as a fallback mirror. It is populated with hard
+links to the archive contents. In CONF.sh, simply supply as
+JIGDOFALLBACKPATH the name of the directory, and as JIGDOFALLBACKURLS
+the URLs under which it will be made available (two separate URLs are
+necessary, one for "Debian" and one for "Non-US").
+
 
 About the hook system
 =====================

debian/control

@@ -8,7 +8,7 @@ Standards-Version: 3.6.0
 Package: debian-cd
 Architecture: all
 Depends: perl, cpp, libdigest-md5-perl, sysutils, apt (>= 0.3.11.1), make, mkisofs | mkhybrid, lynx, grep-dctrl, apt-utils (>= 0.5.4), debootstrap, bc
-Recommends: hfsutils, debianutils (>= 1.6)
+Recommends: hfsutils, debianutils (>= 1.6), jigdo-file (>= 0.7.0)
 Suggests: palo (>= 0.94), binutils-multiarch
 Description: Tools for building (Official) Debian CD set
  Debian-cd is the official tool for building Debian CD set since the potato

tools/jigdo_create

@@ -0,0 +1,44 @@
+#! /bin/sh
+
+# Copyright 2003 Richard Atterer (atterer@d.o)
+# See the README file for the license
+
+# This script is called by the Makefile to create .jigdo and .template
+# files for images. The image data is either supplied in a file or on
+# stdin.
+
+image="$1"    # Input image data, either a filename or "-" for stdin
+jigdo="$2"    # Name of output .jigdo file
+template="$3" # Name of output .template file
+merge="$4"    # Name of file to include in output .jigdo, created by jigdo_header
+
+opts="--label Debian=$MIRROR"
+if test -n "$NONUS"; then
+    opts="$opts --label Non-US=$NONUS"
+fi
+
+if test -n "$JIGDOFALLBACKPATH"; then
+    matchexec='--match-exec=mkdir -p "$JIGDOFALLBACKPATH/$LABEL/$MATCHPATH" && ln -f "$FILE" "$JIGDOFALLBACKPATH/$LABEL/$MATCHPATH$LEAF"'
+else
+    matchexec="--"
+fi
+
+echo $JIGDOCMD make-template \
+  --force \
+  --files-from="$TDIR/jigdofilelist" \
+  --image="$image" --jigdo="$jigdo" --template="$template" \
+  --merge="$merge" \
+  --no-servers-section \
+  --report=noprogress \
+  $opts \
+  "$matchexec"
+
+$JIGDOCMD make-template \
+  --force \
+  --files-from="$TDIR/jigdofilelist" \
+  --image="$image" --jigdo="$jigdo" --template="$template" \
+  --merge="$merge" \
+  --no-servers-section \
+  --report=noprogress \
+  $opts \
+  "$matchexec"

tools/jigdo_header

@@ -0,0 +1,45 @@
+#! /bin/sh
+
+# Copyright 2003 Richard Atterer (atterer@d.o)
+# See the README file for the license
+
+# This script is called by the Makefile to output to stdout data which
+# is to be included in the .jigdo file for each CD/DVD. The data is
+# read by jigdo-file. jigdo-file adds some further lines (a [Jigdo]
+# section and a [Parts] section) before writing everything to the
+# final .jigdo output file.
+
+iso="$1" # Output .iso file
+template="$2" # URL for .template file, can be relative URL
+discinfo="$3" # "Debian GNU/Linux 3.0 r1 "Woody" - Unofficial i386 Binary-1"
+
+# Output image section with info about the file
+echo "[Image]"
+echo "Filename=$iso"
+echo "Template=$template"
+echo "Template-MD5Sum=" # jigdo-file will append the checksum later
+echo "ShortInfo='$discinfo'"
+echo "Info='Generated on `date -R`'"
+
+# If some include URLs were specified, add them here. Typically, this
+# will be used to fetch an up-to-date list of Debian servers.
+if test -n "$JIGDOINCLUDEURLS"; then
+    echo
+    for url in $JIGDOINCLUDEURLS; do
+	echo "[Include $url]"
+    done
+fi
+
+# If some fallbacks were specified, output a servers section with the
+# URLs. The entries of the variable are expected to be already of the
+# form "Label=http://some.url/"
+if test -n "$JIGDOFALLBACKURLS"; then
+    echo
+    echo "[Servers]"
+    for url in $JIGDOFALLBACKURLS; do
+	# The --try-last switch assigns a lower priority to the URL,
+	# so it will only be used if other server entries (without the
+	# --try-last) have already been tried without success.
+	echo "$url --try-last"
+    done
+fi