Add jigdo generation support

CONF.sh

@@ -18,13 +18,16 @@ unset MKISOFS_OPTS
 unset EXCLUDE
 unset SRCEXCLUDE
 unset NORECOMMENDS
+unset DOJIGDO
+unset JIGDOCMD
+unset JIGDOTEMPLATEURL
 
 
 # The debian-cd dir
 # Where I am (hoping I'm in the debian-cd dir)
 export BASEDIR=`pwd`
 
-# Building potato cd set ...
+# Building woody cd set ...
 export CODENAME=woody
 
 # Version number, "2.2 r0", "2.2 r1" etc.
@@ -124,3 +127,24 @@ export APTTMP=/ftp/tmp/apt
 # Set this if only the required (and NOT the recommended/suggested) packages
 # should be added on CDs when a package is added on the CD.
 #export NORECOMMENDS=1
+
+# Produce jigdo files:
+# 0/unset = Don't do jigdo at all, produce only the full iso image.
+# 1 = Produce both the iso image and jigdo stuff.
+# 2 = Produce ONLY jigdo stuff by piping mkisofs directly into jigdo-file,
+#     no temporary iso image is created (saves lots of disk space).
+#     NOTE: The no-temp-iso will not work for (at least) alpha and powerpc
+#     since they need the actual .iso to make it bootable. For these archs,
+#     the temp-iso will be generated, but deleted again immediately after the
+#     jigdo stuff is made; needs temporary space as big as the biggest image.
+#export DOJIGDO=2
+#
+# jigdo-file command & options
+# Note: building the cache takes hours, so keep it around for the next run
+#export JIGDOCMD="/usr/local/bin/jigdo-file --cache=$HOME/jigdo-cache.db"
+#
+# HTTP/FTP URL for directory where you intend to make the templates available.
+# %ARCH%, if present, will be replaced by $ARCH (or "source"). This only goes
+# in the .jigdo files, which you can edit easily if you want.
+# No trailing slash.
+#export JIGDOTEMPLATEURL="http://this-guy-didnt-configure-debiancd-correctly.com/debian-cd/templates/3.0BETA/%ARCH%"

Makefile

@@ -61,6 +61,9 @@ endif
 ifndef BOOTDISKS
 export BOOTDISKS=$(MIRROR)/dists/$(CODENAME)/main/disks-$(ARCH)
 endif
+ifndef DOJIGDO
+export DOJIGDO=0
+endif
 
 ## Internal variables  
 apt=$(BASEDIR)/tools/apt-selection
@@ -142,6 +145,17 @@ ifndef NONUS
 ok=false
 endif
 endif
+# If we do jigdo, we need a command and a URL. If not, make sure we won't.
+ifneq "$(DOJIGDO)" "0"
+ifndef JIGDOCMD
+ok=false
+endif
+ifndef JIGDOTEMPLATEURL
+ok=false
+endif
+else
+export JIGDOCMD=false
+endif
 
 default:
 	@echo "Please refer to the README file for more information"
@@ -688,40 +702,151 @@ $(SDIR)/CD1/dists/$(CODENAME)-secured:
 		$(add_secured); \
 	done
 
+# Make file list for jigdo (if DOJIGDO>0)
+# "Fake" depend on the unstable Packages.gz to make sure we only regenerate
+# this list when really necessary (saves many minutes per run).
+# Don't depend on anything else as this will not work as intended, so
+# make $(TDIR) ourselves just to be sure.
+$(TDIR)/jigdofilelist: $(MIRROR)/dists/unstable/main/binary-i386/Packages.gz
+	@echo "Generating file list for jigdo (if requested) ..."
+	$(Q)set -e; \
+	if [ "$(DOJIGDO)" != 0 ]; then \
+		mkdir -p $(TDIR); \
+		find $(MIRROR)//dists $(MIRROR)//doc $(MIRROR)//indices \
+		     $(MIRROR)//pool $(MIRROR)//project $(MIRROR)//tools \
+		     -type f \
+		| egrep -v '/README|INDEX$$|/Maintainers|/Release$$|/debian-keyring\.tar\.gz$$|/ls-lR|//doc/[^/]+/?[^/]*\.(txt|html)$$' \
+		> $(TDIR)/jigdofilelist; \
+		if [ -n "$(NONUS)" ]; then \
+			find $(NONUS)// -type f \
+			| egrep -v '/README|INDEX$$|/Maintainers|/Release$$|/debian-keyring\.tar\.gz$$|/ls-lR|//doc/[^/]+/?[^/]*\.(txt|html)$$' \
+			>> $(TDIR)/jigdofilelist; \
+		fi; \
+	fi
 
 # Generates all the images
 images: bin-images src-images
-bin-images: ok bin-md5list $(OUT)
+
+# DOJIGDO postboot actions   (source has the appropriate subset)
+#    0      no     isofile
+#    0      yes    isofile  post
+#    1      no     isofile        jigdo            jigdoadd
+#    1      yes    isofile  post  jigdo            jigdoadd
+#    2      no                           isojigdo  jigdoadd
+#    2      yes    isofile  post  jigdo            jigdoadd  rmiso 
+#
+bin-images: ok bin-md5list $(OUT) $(TDIR)/jigdofilelist
 	@echo "Generating the binary iso images ..."
 	$(Q)set -e; \
 	 for file in $(BDIR)/*.packages; do \
 		dir=$${file%%.packages}; \
 		n=$${dir##$(BDIR)/}; \
+		num=$$n; \
 		dir=$(BDIR)/CD$$n; \
 		cd $$dir/..; \
 		opts=`cat $(BDIR)/$$n.mkisofs_opts`; \
 		volid=`cat $(BDIR)/$$n.volid`; \
 		rm -f $(OUT)/$(CODENAME)-$(ARCH)-$$n.raw; \
+		if [ "$(DOJIGDO)" != "2" -o -f $(BASEDIR)/tools/boot/$(CODENAME)/post-boot-$(ARCH) ]; then \
 			$(MKISOFS) $(MKISOFS_OPTS) -V "$$volid" \
 			  -o $(OUT)/$(CODENAME)-$(ARCH)-$$n.raw $$opts CD$$n ; \
 			if [ -f $(BASEDIR)/tools/boot/$(CODENAME)/post-boot-$(ARCH) ]; then \
 				$(BASEDIR)/tools/boot/$(CODENAME)/post-boot-$(ARCH) $$n $$dir \
 				 $(OUT)/$(CODENAME)-$(ARCH)-$$n.raw; \
-		fi \
+			fi; \
+			if [ "$(DOJIGDO)" != "0" ]; then \
+				$(JIGDOCMD) make-template --force \
+				  --files-from=$(TDIR)/jigdofilelist \
+				  --image=$(OUT)/$(CODENAME)-$(ARCH)-$$n.raw \
+				  --jigdo=$(OUT)/$(CODENAME)-$(ARCH)-$$n.jigdo \
+				  --template=$(OUT)/$(CODENAME)-$(ARCH)-$$n.template \
+				  --label Non-US="$(NONUS)" \
+				  --label Debian="$(MIRROR)" \
+				  --no-image-section --no-servers-section \
+				  --report=noprogress; \
+			fi; \
+		else \
+			$(MKISOFS) $(MKISOFS_OPTS) -V "$$volid" \
+			  $$opts CD$$n \
+			| $(JIGDOCMD) make-template --force \
+				  --files-from=$(TDIR)/jigdofilelist \
+				  --image=- \
+				  --jigdo=$(OUT)/$(CODENAME)-$(ARCH)-$$n.jigdo \
+				  --template=$(OUT)/$(CODENAME)-$(ARCH)-$$n.template \
+				  --label Non-US="$(NONUS)" \
+				  --label Debian="$(MIRROR)" \
+				  --no-image-section --no-servers-section \
+				  --report=noprogress; \
+		fi; \
+		if [ "$(DOJIGDO)" != "0" ]; then \
+			echo "" >> $(OUT)/$(CODENAME)-$(ARCH)-$$n.jigdo; \
+			echo "[Image]" \
+				>> $(OUT)/$(CODENAME)-$(ARCH)-$$n.jigdo; \
+			echo "Filename=debian-`echo $(DEBVERSION) | sed -e 's/[. ]//g'`-$(ARCH)-binary-$$n.iso" \
+				>> $(OUT)/$(CODENAME)-$(ARCH)-$$n.jigdo; \
+			echo "Template=`echo "$(JIGDOTEMPLATEURL)" | sed -e 's|%ARCH%|$(ARCH)|g'`/$(CODENAME)-$(ARCH)-$$n.template" \
+				>> $(OUT)/$(CODENAME)-$(ARCH)-$$n.jigdo; \
+			echo "ShortInfo='"$(BINDISKINFOND)" CD'" \
+				>> $(OUT)/$(CODENAME)-$(ARCH)-$$n.jigdo; \
+			echo "Info='Generated on `date -R`'" \
+				>> $(OUT)/$(CODENAME)-$(ARCH)-$$n.jigdo; \
+		fi; \
+		if [ "$(DOJIGDO)" = "2" ]; then \
+			rm -f $(OUT)/$(CODENAME)-$(ARCH)-$$n.raw; \
+		fi; \
 	done
-src-images: ok src-md5list $(OUT)
+src-images: ok src-md5list $(OUT) $(TDIR)/jigdofilelist
 	@echo "Generating the source iso images ..."
 	$(Q)set -e; \
 	 for file in $(SDIR)/*.sources; do \
 		dir=$${file%%.sources}; \
 		n=$${dir##$(SDIR)/}; \
+		num=$$n; \
 		dir=$(SDIR)/CD$$n; \
 		cd $$dir/..; \
 		opts=`cat $(SDIR)/$$n.mkisofs_opts`; \
 		volid=`cat $(SDIR)/$$n.volid`; \
 		rm -f $(OUT)/$(CODENAME)-src-$$n.raw; \
+		if [ "$(DOJIGDO)" != "2" ]; then \
 			$(MKISOFS) $(MKISOFS_OPTS) -V "$$volid" \
 			  -o $(OUT)/$(CODENAME)-src-$$n.raw $$opts CD$$n ; \
+			if [ "$(DOJIGDO)" != "0" ]; then \
+				$(JIGDOCMD) make-template --force \
+				  --files-from=$(TDIR)/jigdofilelist \
+				  --image=$(OUT)/$(CODENAME)-src-$$n.raw \
+				  --jigdo=$(OUT)/$(CODENAME)-src-$$n.jigdo \
+				  --template=$(OUT)/$(CODENAME)-src-$$n.template \
+				  --label Non-US="$(NONUS)" \
+				  --label Debian="$(MIRROR)" \
+				  --no-image-section --no-servers-section \
+				  --report=noprogress; \
+			fi; \
+		else \
+			$(MKISOFS) $(MKISOFS_OPTS) -V "$$volid" \
+			  $$opts CD$$n \
+			| $(JIGDOCMD) make-template --force \
+				  --files-from=$(TDIR)/jigdofilelist \
+				  --image=- \
+				  --jigdo=$(OUT)/$(CODENAME)-src-$$n.jigdo \
+				  --template=$(OUT)/$(CODENAME)-src-$$n.template \
+				  --label Non-US="$(NONUS)" \
+				  --label Debian="$(MIRROR)" \
+				  --no-image-section --no-servers-section \
+				  --report=noprogress; \
+		fi; \
+		if [ "$(DOJIGDO)" != "0" ]; then \
+			echo "" >> $(OUT)/$(CODENAME)-src-$$n.jigdo; \
+			echo "[Image]" \
+				>> $(OUT)/$(CODENAME)-src-$$n.jigdo; \
+			echo "Filename=debian-`echo $(DEBVERSION) | sed -e 's/[. ]//g'`-source-$$n.iso" \
+				>> $(OUT)/$(CODENAME)-src-$$n.jigdo; \
+			echo "Template=`echo "$(JIGDOTEMPLATEURL)" | sed -e 's|%ARCH%|source|g'`/$(CODENAME)-src-$$n.template" \
+				>> $(OUT)/$(CODENAME)-src-$$n.jigdo; \
+			echo "ShortInfo='"$(SRCDISKINFOND)" CD'" \
+				>> $(OUT)/$(CODENAME)-src-$$n.jigdo; \
+			echo "Info='Generated on `date -R`'" \
+				>> $(OUT)/$(CODENAME)-src-$$n.jigdo; \
+		fi; \
 	done
 
 # Generate the *.list files for the Pseudo Image Kit
@@ -754,12 +879,24 @@ src-image: ok src-md5list $(OUT)
 	  -o $(OUT)/$(CODENAME)-src-$(CD).raw $$opts CD$(CD)
 
 
-#Calculate the md5sums for the images
+#Calculate the md5sums for the images (if available), or get from templates
 imagesums:
 	$(Q)cd $(OUT); :> MD5SUMS; for file in `find * -name \*.raw`; do \
 		md5sum $$file >>MD5SUMS; \
+	done; \
+	for file in `find * -name \*.template`; do \
+		if [ "`tail --bytes=29 "$$file" | head --bytes=1 | od -tx1 -An | sed -e 's/ //g'`" != 01 ]; then \
+			echo "Possibly invalid template $$file"; exit 1; \
+		fi; \
+		grep -q " $${file%%.template}.raw"'$$' MD5SUMS \
+		 || echo "`tail --bytes=22 "$$file" | head --bytes=16 | od -tx1 -An | sed -e 's/ //g'`  $${file%%.template}.raw" >>MD5SUMS; \
 	done
 
+# Likewise, the file size can be extracted from the .template with:
+# tail --bytes=28 $$file | head --bytes=6 | od -tx1 -An \
+#  | tr ' abcdef' '\nABCDEF' | tac | tr '\n' ' ' \
+#  | sed -e 's/ //g; s/^.*$/ibase=16 & /' | tr ' ' '\n' | bc
+
 ## MISC TARGETS ##
 
 tasks: ok $(BASEDIR)/data/$(CODENAME)/master

README

@@ -26,6 +26,7 @@ Software :
 - lynx (for text version of README.html)
 - if you want to generate .list files: tempfile, as included in debianutils
   >= 1.6 (not absolutely necessary, workaround described in tools/pi-makelist)
+- if you want to generate jigdo files: jigdo-file (see below)
 Other :
 - lots of free space on your disks
 - a Debian mirror (on a partition where you can write, if you can't write
@@ -253,6 +254,33 @@ sources from http://www.fokus.gmd.de/research/cc/glone/employees/joerg.schilling
 Alternatively (and certainly simpler for people using Debian system), the
 isoinfo binary is in the mkisofs package (since Debian potato at least).
 
+Jigdo
+-----
+People wanting to offer the CD images in jigdo format (Jigsaw Download,
+the successor to the Pseudo-Image Kit), can indicate so by setting the
+DOJIGDO and related variables in CONF.sh. So this is not a target itself,
+but a modification of the "images" targets. You can choose only iso
+generation (default), only jigdo generation (for highly reduced disk
+usage), or both iso and jigdo generation.
+
+Information on jigdo is available at  http://www.debian.org/CD/jigdo-cd/
+
+To generate the jigdo files and templates, the "jigdo-file" program is
+needed, which is available via that same URL.
+
+Jigdo-file creates and maintains a cache/database file with checksums of
+all files on your Debian mirror. The first time, this may take hours to be
+generated (use "top" to see what's going on), so it's wise to keep the
+cache in your homedir and not delete it ever. 
+
+Note that jigdo-file can easily use 60+ MB of working memory, so don't use
+this on machines with less than 128 MB RAM.
+
+The MD5SUMS file generated by the "imagesums" target will contain the MD5
+checksums of all generated images, regardless of the DOJIGDO setting. If
+no full iso image is available, the MD5sum will be extracted from the
+.template file. A note in the Makefile shows how the original file size
+can be extracted from the .template in a similar way.
 
 About the hook system
 =====================

debian/CONF.sh

@@ -14,6 +14,10 @@ unset MKISOFS_OPTS
 unset EXCLUDE
 unset SRCEXCLUDE
 unset NORECOMMENDS
+unset DOJIGDO
+unset JIGDOCMD
+unset JIGDOTEMPLATEURL
+
 
 # The debian-cd dir
 export BASEDIR=/usr/share/debian-cd
@@ -118,3 +122,24 @@ export APTTMP=/home/ftp/tmp/apt
 # Set this if only the required (and NOT the recommended/suggested) packages
 # should be added on CDs when a package is added on the CD.
 #export NORECOMMENDS=1
+
+# Produce jigdo files:
+# 0/unset = Don't do jigdo at all, produce only the full iso image.
+# 1 = Produce both the iso image and jigdo stuff.
+# 2 = Produce ONLY jigdo stuff by piping mkisofs directly into jigdo-file,
+#     no temporary iso image is created (saves lots of disk space).
+#     NOTE: The no-temp-iso will not work for (at least) alpha and powerpc
+#     since they need the actual .iso to make it bootable. For these archs,
+#     the temp-iso will be generated, but deleted again immediately after the
+#     jigdo stuff is made; needs temporary space as big as the biggest image.
+#export DOJIGDO=2
+#
+# jigdo-file command & options
+# Note: building the cache takes hours, so keep it around for the next run
+#export JIGDOCMD="/usr/local/bin/jigdo-file --cache=$HOME/jigdo-cache.db"
+#
+# HTTP/FTP URL for directory where you intend to make the templates available.
+# %ARCH%, if present, will be replaced by $ARCH (or "source"). This only goes
+# in the .jigdo files, which you can edit easily if you want.
+# No trailing slash.
+#export JIGDOTEMPLATEURL="http://this-guy-didnt-configure-debiancd-correctly.com/debian-cd/templates/3.0BETA/%ARCH%"

debian/changelog

@@ -1,6 +1,7 @@
 debian-cd (2.2.13) unstable; urgency=low
 
   * UNRELEASED. 
+  * Add jigdo generation support (see README).
 
  -- Raphael Hertzog <hertzog@debian.org>  Sun, 20 Jan 2002 15:42:29 +0100