From e6440d5a07c8ef8d3bbaaa1f34804375b105ba1c Mon Sep 17 00:00:00 2001 From: Richard Atterer Date: Thu, 18 Dec 2003 10:22:38 +0000 Subject: [PATCH] Added code/docs for automatic generation of fallback directories during jigdo generation --- CONF.sh | 46 +++++++++++++++++---- Makefile | 100 ++++++++++++++------------------------------- README | 27 ++++++++++++ debian/control | 2 +- tools/jigdo_create | 44 ++++++++++++++++++++ tools/jigdo_header | 45 ++++++++++++++++++++ 6 files changed, 187 insertions(+), 77 deletions(-) create mode 100755 tools/jigdo_create create mode 100755 tools/jigdo_header diff --git a/CONF.sh b/CONF.sh index f9c9ce71..ee337809 100644 --- a/CONF.sh +++ b/CONF.sh @@ -26,6 +26,9 @@ unset NOSUGGESTS || true unset DOJIGDO || true unset JIGDOCMD || true unset JIGDOTEMPLATEURL || true +unset JIGDOFALLBACKURLS || true +unset JIGDOINCLUDEURLS || true +unset JIGDOSCRIPT || true unset DEFBINSIZE || true unset DEFSRCSIZE || true unset FASTSUMS || true @@ -179,18 +182,47 @@ export DEFSRCSIZE=635 # Note: building the cache takes hours, so keep it around for the next run #export JIGDOCMD="/usr/local/bin/jigdo-file --cache=$HOME/jigdo-cache.db" # -# HTTP/FTP URL for directory where you intend to make the templates available. -# %ARCH%, if present, will be replaced by $ARCH (or "source"). This only goes -# in the .jigdo files, which you can edit easily if you want. -# No trailing slash. -#export JIGDOTEMPLATEURL="http://this-guy-didnt-configure-debiancd-correctly.com/debian-cd/templates/3.0BETA/%ARCH%" +# HTTP/FTP URL for directory where you intend to make the templates +# available. You should not need to change this; the default value "" +# means "template in same dir as the .jigdo file", which is usually +# correct. If it is non-empty, it needs a trailing slash. "%ARCH%" +# will be substituted by the current architecture. +#export JIGDOTEMPLATEURL="" +# +# Name of a directory on disc to create data for a fallback server in. +# Should later be made available by you at the URL given in +# JIGDOFALLBACKURLS. In the directory, two subdirs named "Debian" and +# "Non-US" will be created, and filled with hard links to the actual +# files in your FTP archive. Because of the hard links, the dir must +# be on the same partition as the FTP archive! If unset, no fallback +# data is created, which may cause problems - see README. +#export JIGDOFALLBACKPATH="$(OUT)/snapshot/" +# +# Space-separated list of label->URL mappings for "jigdo fallback +# server(s)" to add to .jigdo file. If unset, no fallback URL is +# added, which may cause problems - see README. +#export JIGDOFALLBACKURLS="Debian=http://myserver/snapshot/Debian/ Non-US=http://myserver/snapshot/Non-US/" +# +# Space-separated list of "include URLs" to add to the .jigdo file. +# The included files are used to provide an up-to-date list of Debian +# mirrors to the jigdo _GUI_application_ (_jigdo-lite_ doesn't support +# "[Include ...]"). +export JIGDOINCLUDEURLS="http://cdimage.debian.org/debian-cd/debian-servers.jigdo" +# +# $JIGDOTEMPLATEURL and $JIGDOINCLUDEURLS are passed to +# "tools/jigdo_header", which is used by default to generate the +# [Image] and [Servers] sections of the .jigdo file. You can provide +# your own script if you need the .jigdo file to contain different +# data. +#export JIGDOSCRIPT="myscript" # If set, use the md5sums from the main archive, rather than calculating # them locally #export FASTSUMS=1 -# a couple of things used by publish_cds, so it can tweak the jigdo files, -# and knows where to put the results +# A couple of things used only by publish_cds, so it can tweak the +# jigdo files, and knows where to put the results. +# You need to run publish_cds manually, it is not run by the Makefile. export PUBLISH_URL="http://cdimage.debian.org/jigdo-area" export PUBLISH_NONUS_URL="http://non-US.cdimage.debian.org/jigdo-area" export PUBLISH_PATH="/home/jigdo-area/" diff --git a/Makefile b/Makefile index 72dcac54..80378334 100755 --- a/Makefile +++ b/Makefile @@ -24,17 +24,17 @@ ifndef CAPCODENAME CAPCODENAME:=$(shell perl -e "print ucfirst("$(CODENAME)")") endif ifndef BINDISKINFO -BINDISKINFO="Debian GNU/Linux $(DEBVERSION) \"$(CAPCODENAME)\" - $(OFFICIAL) $(ARCH) Binary-$$num ($$DATE)" +export BINDISKINFO="Debian GNU/Linux $(DEBVERSION) \"$(CAPCODENAME)\" - $(OFFICIAL) $(ARCH) Binary-$$num ($$DATE)" endif ifndef SRCDISKINFO -SRCDISKINFO="Debian GNU/Linux $(DEBVERSION) \"$(CAPCODENAME)\" - $(OFFICIAL) Source-$$num ($$DATE)" +export SRCDISKINFO="Debian GNU/Linux $(DEBVERSION) \"$(CAPCODENAME)\" - $(OFFICIAL) Source-$$num ($$DATE)" endif # ND=No-Date versions for README ifndef BINDISKINFOND -BINDISKINFOND="Debian GNU/Linux $(DEBVERSION) \"$(CAPCODENAME)\" - $(OFFICIAL) $(ARCH) Binary-$$num" +export BINDISKINFOND="Debian GNU/Linux $(DEBVERSION) \"$(CAPCODENAME)\" - $(OFFICIAL) $(ARCH) Binary-$$num" endif ifndef SRCDISKINFOND -SRCDISKINFOND="Debian GNU/Linux $(DEBVERSION) \"$(CAPCODENAME)\" - $(OFFICIAL) Source-$$num" +export SRCDISKINFOND="Debian GNU/Linux $(DEBVERSION) \"$(CAPCODENAME)\" - $(OFFICIAL) Source-$$num" endif ifndef BINVOLID ifeq ($(ARCH),powerpc) @@ -66,6 +66,9 @@ endif ifndef DOJIGDO export DOJIGDO=0 endif +ifndef JIGDOSCRIPT +JIGDOSCRIPT=$(BASEDIR)/tools/jigdo_header +endif ifndef UDEB_INCLUDE # Netinst/businesscard CD should have udeb_include file by default @@ -165,9 +168,6 @@ ifneq "$(DOJIGDO)" "0" ifndef JIGDOCMD @echo JIGDOCMD undefined -- set up CONF.sh; false endif -ifndef JIGDOTEMPLATEURL - @echo JIGDOTEMPLATEURL undefined -- set up CONF.sh; false -endif else export JIGDOCMD=false endif @@ -846,18 +846,11 @@ bin-images: ok bin-md5list $(OUT) $(TDIR)/jigdofilelist volid=`cat $(BDIR)/$$n.volid`; \ rm -f $(OUT)/$(CODENAME)-$(ARCH)-$$n.raw; \ if [ "$(DOJIGDO)" != "0" ]; then \ - echo "[Image]" \ + $(JIGDOSCRIPT) \ + "debian-`echo $(DEBVERSION) | sed -e 's/[. ]//g'`-$(ARCH)-binary-$$n.iso" \ + "`echo "$(JIGDOTEMPLATEURL)" | sed -e 's|%ARCH%|$(ARCH)|g'`$(CODENAME)-$(ARCH)-$$n.template" \ + $(BINDISKINFOND) \ > $(TDIR)/$(CODENAME)-$(ARCH).jigdo; \ - echo "Filename=debian-`echo $(DEBVERSION) | sed -e 's/[. ]//g'`-$(ARCH)-binary-$$n.iso" \ - >> $(TDIR)/$(CODENAME)-$(ARCH).jigdo; \ - echo "Template=`echo "$(JIGDOTEMPLATEURL)" | sed -e 's|%ARCH%|$(ARCH)|g'`/$(CODENAME)-$(ARCH)-$$n.template" \ - >> $(TDIR)/$(CODENAME)-$(ARCH).jigdo; \ - echo "Template-MD5Sum=" \ - >> $(TDIR)/$(CODENAME)-$(ARCH).jigdo; \ - echo "ShortInfo='"$(BINDISKINFOND)" CD'" \ - >> $(TDIR)/$(CODENAME)-$(ARCH).jigdo; \ - echo "Info='Generated on `date -R`'" \ - >> $(TDIR)/$(CODENAME)-$(ARCH).jigdo; \ fi; \ if [ "$(DOJIGDO)" != "2" -o -f $(BASEDIR)/tools/boot/$(CODENAME)/post-boot-$(ARCH) ]; then \ $(MKISOFS) $(MKISOFS_OPTS) -V "$$volid" \ @@ -867,30 +860,18 @@ bin-images: ok bin-md5list $(OUT) $(TDIR)/jigdofilelist $(OUT)/$(CODENAME)-$(ARCH)-$$n.raw; \ fi; \ if [ "$(DOJIGDO)" != "0" ]; then \ - $(JIGDOCMD) make-template --force \ - --files-from=$(TDIR)/jigdofilelist \ - --image=$(OUT)/$(CODENAME)-$(ARCH)-$$n.raw \ - --jigdo=$(OUT)/$(CODENAME)-$(ARCH)-$$n.jigdo \ - --template=$(OUT)/$(CODENAME)-$(ARCH)-$$n.template \ - --label Non-US="$(NONUS)" \ - --label Debian="$(MIRROR)" \ - --merge="$(TDIR)/$(CODENAME)-$(ARCH).jigdo" \ - --no-servers-section \ - --report=noprogress; \ + $(BASEDIR)/tools/jigdo_create "$(OUT)/$(CODENAME)-$(ARCH)-$$n.raw" \ + "$(OUT)/$(CODENAME)-$(ARCH)-$$n.jigdo" \ + "$(OUT)/$(CODENAME)-$(ARCH)-$$n.template" \ + "$(TDIR)/$(CODENAME)-$(ARCH).jigdo"; \ fi; \ else \ $(MKISOFS) $(MKISOFS_OPTS) -V "$$volid" \ $$opts CD$$n \ - | $(JIGDOCMD) make-template --force \ - --files-from=$(TDIR)/jigdofilelist \ - --image=- \ - --jigdo=$(OUT)/$(CODENAME)-$(ARCH)-$$n.jigdo \ - --template=$(OUT)/$(CODENAME)-$(ARCH)-$$n.template \ - --label Non-US="$(NONUS)" \ - --label Debian="$(MIRROR)" \ - --merge="$(TDIR)/$(CODENAME)-$(ARCH).jigdo" \ - --no-servers-section \ - --report=noprogress; \ + | $(BASEDIR)/tools/jigdo_create "-" \ + "$(OUT)/$(CODENAME)-$(ARCH)-$$n.jigdo" \ + "$(OUT)/$(CODENAME)-$(ARCH)-$$n.template" \ + "$(TDIR)/$(CODENAME)-$(ARCH).jigdo"; \ fi; \ if [ "$(DOJIGDO)" = "2" ]; then \ rm -f $(OUT)/$(CODENAME)-$(ARCH)-$$n.raw; \ @@ -910,47 +891,28 @@ src-images: ok src-md5list $(OUT) $(TDIR)/jigdofilelist volid=`cat $(SDIR)/$$n.volid`; \ rm -f $(OUT)/$(CODENAME)-src-$$n.raw; \ if [ "$(DOJIGDO)" != "0" ]; then \ - echo "[Image]" \ + $(JIGDOSCRIPT) \ + "debian-`echo $(DEBVERSION) | sed -e 's/[. ]//g'`-source-$$n.iso" \ + "`echo "$(JIGDOTEMPLATEURL)" | sed -e 's|%ARCH%|$(ARCH)|g'`$(CODENAME)-src-$$n.template" \ + $(SRCDISKINFOND) \ > $(TDIR)/$(CODENAME)-src.jigdo; \ - echo "Filename=debian-`echo $(DEBVERSION) | sed -e 's/[. ]//g'`-source-$$n.iso" \ - >> $(TDIR)/$(CODENAME)-src.jigdo; \ - echo "Template=`echo "$(JIGDOTEMPLATEURL)" | sed -e 's|%ARCH%|source|g'`/$(CODENAME)-src-$$n.template" \ - >> $(TDIR)/$(CODENAME)-src.jigdo; \ - echo "Template-MD5Sum=" \ - >> $(TDIR)/$(CODENAME)-src.jigdo; \ - echo "ShortInfo='"$(SRCDISKINFOND)" CD'" \ - >> $(TDIR)/$(CODENAME)-src.jigdo; \ - echo "Info='Generated on `date -R`'" \ - >> $(TDIR)/$(CODENAME)-src.jigdo; \ fi; \ if [ "$(DOJIGDO)" != "2" ]; then \ $(MKISOFS) $(MKISOFS_OPTS) -V "$$volid" \ -o $(OUT)/$(CODENAME)-src-$$n.raw $$opts CD$$n ; \ if [ "$(DOJIGDO)" != "0" ]; then \ - $(JIGDOCMD) make-template --force \ - --files-from=$(TDIR)/jigdofilelist \ - --image=$(OUT)/$(CODENAME)-src-$$n.raw \ - --jigdo=$(OUT)/$(CODENAME)-src-$$n.jigdo \ - --template=$(OUT)/$(CODENAME)-src-$$n.template \ - --label Non-US="$(NONUS)" \ - --label Debian="$(MIRROR)" \ - --merge="$(TDIR)/$(CODENAME)-src.jigdo" \ - --no-servers-section \ - --report=noprogress; \ + $(BASEDIR)/tools/jigdo_create "$(OUT)/$(CODENAME)-src-$$n.raw" \ + "$(OUT)/$(CODENAME)-src-$$n.jigdo" \ + "$(OUT)/$(CODENAME)-src-$$n.template" \ + "$(TDIR)/$(CODENAME)-src.jigdo"; \ fi; \ else \ $(MKISOFS) $(MKISOFS_OPTS) -V "$$volid" \ $$opts CD$$n \ - | $(JIGDOCMD) make-template --force \ - --files-from=$(TDIR)/jigdofilelist \ - --image=- \ - --jigdo=$(OUT)/$(CODENAME)-src-$$n.jigdo \ - --template=$(OUT)/$(CODENAME)-src-$$n.template \ - --label Non-US="$(NONUS)" \ - --label Debian="$(MIRROR)" \ - --merge="$(TDIR)/$(CODENAME)-src.jigdo" \ - --no-servers-section \ - --report=noprogress; \ + | $(BASEDIR)/tools/jigdo_create "-" \ + "$(OUT)/$(CODENAME)-src-$$n.jigdo" \ + "$(OUT)/$(CODENAME)-src-$$n.template" \ + "$(TDIR)/$(CODENAME)-src.jigdo"; \ fi; \ done rm -f "$(TDIR)/$(CODENAME)-src.jigdo" diff --git a/README b/README index a39181f1..caaf00aa 100644 --- a/README +++ b/README @@ -317,6 +317,33 @@ no full iso image is available, the MD5sum will be extracted from the .template file. A note in the Makefile shows how the original file size can be extracted from the .template in a similar way. +About jigdo "fallback servers": + +jigdo works by downloading individual packages and other files from a +normal Debian mirror, and using them to regenerate a CD/DVD image. +However, the content of Debian mirrors changes over time, files are +added and removed. But jigdo must have access to all files needed for +the image it has to regenerate, even those that have been removed from +the normal Debian mirrors. + +A fallback server contains a backup of the Debian FTP space for the +moment the .jigdo files were generated. This backup is made available +under a certain URL which is written to the .jigdo files. jigdo will +*only* revert to the fallback server after an unsuccessful attempt to +retrieve a file from the normal user-selected Debian mirror, so the +bandwidth requirements are modest. + +A fallback is even necessary for .jigdo files of the stable release, +because some files (typically documentation or boot floppies) can +change at any time. + +debian-cd allows you to automatically create a directory on disc which +is suitable for use as a fallback mirror. It is populated with hard +links to the archive contents. In CONF.sh, simply supply as +JIGDOFALLBACKPATH the name of the directory, and as JIGDOFALLBACKURLS +the URLs under which it will be made available (two separate URLs are +necessary, one for "Debian" and one for "Non-US"). + About the hook system ===================== diff --git a/debian/control b/debian/control index c9da77e0..f28693b2 100644 --- a/debian/control +++ b/debian/control @@ -8,7 +8,7 @@ Standards-Version: 3.6.0 Package: debian-cd Architecture: all Depends: perl, cpp, libdigest-md5-perl, sysutils, apt (>= 0.3.11.1), make, mkisofs | mkhybrid, lynx, grep-dctrl, apt-utils (>= 0.5.4), debootstrap, bc -Recommends: hfsutils, debianutils (>= 1.6) +Recommends: hfsutils, debianutils (>= 1.6), jigdo-file (>= 0.7.0) Suggests: palo (>= 0.94), binutils-multiarch Description: Tools for building (Official) Debian CD set Debian-cd is the official tool for building Debian CD set since the potato diff --git a/tools/jigdo_create b/tools/jigdo_create new file mode 100755 index 00000000..928a1a79 --- /dev/null +++ b/tools/jigdo_create @@ -0,0 +1,44 @@ +#! /bin/sh + +# Copyright 2003 Richard Atterer (atterer@d.o) +# See the README file for the license + +# This script is called by the Makefile to create .jigdo and .template +# files for images. The image data is either supplied in a file or on +# stdin. + +image="$1" # Input image data, either a filename or "-" for stdin +jigdo="$2" # Name of output .jigdo file +template="$3" # Name of output .template file +merge="$4" # Name of file to include in output .jigdo, created by jigdo_header + +opts="--label Debian=$MIRROR" +if test -n "$NONUS"; then + opts="$opts --label Non-US=$NONUS" +fi + +if test -n "$JIGDOFALLBACKPATH"; then + matchexec='--match-exec=mkdir -p "$JIGDOFALLBACKPATH/$LABEL/$MATCHPATH" && ln -f "$FILE" "$JIGDOFALLBACKPATH/$LABEL/$MATCHPATH$LEAF"' +else + matchexec="--" +fi + +echo $JIGDOCMD make-template \ + --force \ + --files-from="$TDIR/jigdofilelist" \ + --image="$image" --jigdo="$jigdo" --template="$template" \ + --merge="$merge" \ + --no-servers-section \ + --report=noprogress \ + $opts \ + "$matchexec" + +$JIGDOCMD make-template \ + --force \ + --files-from="$TDIR/jigdofilelist" \ + --image="$image" --jigdo="$jigdo" --template="$template" \ + --merge="$merge" \ + --no-servers-section \ + --report=noprogress \ + $opts \ + "$matchexec" diff --git a/tools/jigdo_header b/tools/jigdo_header new file mode 100755 index 00000000..581a3500 --- /dev/null +++ b/tools/jigdo_header @@ -0,0 +1,45 @@ +#! /bin/sh + +# Copyright 2003 Richard Atterer (atterer@d.o) +# See the README file for the license + +# This script is called by the Makefile to output to stdout data which +# is to be included in the .jigdo file for each CD/DVD. The data is +# read by jigdo-file. jigdo-file adds some further lines (a [Jigdo] +# section and a [Parts] section) before writing everything to the +# final .jigdo output file. + +iso="$1" # Output .iso file +template="$2" # URL for .template file, can be relative URL +discinfo="$3" # "Debian GNU/Linux 3.0 r1 "Woody" - Unofficial i386 Binary-1" + +# Output image section with info about the file +echo "[Image]" +echo "Filename=$iso" +echo "Template=$template" +echo "Template-MD5Sum=" # jigdo-file will append the checksum later +echo "ShortInfo='$discinfo'" +echo "Info='Generated on `date -R`'" + +# If some include URLs were specified, add them here. Typically, this +# will be used to fetch an up-to-date list of Debian servers. +if test -n "$JIGDOINCLUDEURLS"; then + echo + for url in $JIGDOINCLUDEURLS; do + echo "[Include $url]" + done +fi + +# If some fallbacks were specified, output a servers section with the +# URLs. The entries of the variable are expected to be already of the +# form "Label=http://some.url/" +if test -n "$JIGDOFALLBACKURLS"; then + echo + echo "[Servers]" + for url in $JIGDOFALLBACKURLS; do + # The --try-last switch assigns a lower priority to the URL, + # so it will only be used if other server entries (without the + # --try-last) have already been tried without success. + echo "$url --try-last" + done +fi \ No newline at end of file