From b7582144e31139f765f63c5477a59670438b53bc Mon Sep 17 00:00:00 2001 From: Anne Bezemer Date: Mon, 28 Jan 2002 23:00:17 +0000 Subject: [PATCH] Add jigdo generation support --- CONF.sh | 26 +++++++- Makefile | 163 +++++++++++++++++++++++++++++++++++++++++++---- README | 28 ++++++++ debian/CONF.sh | 25 ++++++++ debian/changelog | 1 + 5 files changed, 229 insertions(+), 14 deletions(-) diff --git a/CONF.sh b/CONF.sh index 3ef513f5..2dbafa54 100644 --- a/CONF.sh +++ b/CONF.sh @@ -18,13 +18,16 @@ unset MKISOFS_OPTS unset EXCLUDE unset SRCEXCLUDE unset NORECOMMENDS +unset DOJIGDO +unset JIGDOCMD +unset JIGDOTEMPLATEURL # The debian-cd dir # Where I am (hoping I'm in the debian-cd dir) export BASEDIR=`pwd` -# Building potato cd set ... +# Building woody cd set ... export CODENAME=woody # Version number, "2.2 r0", "2.2 r1" etc. @@ -124,3 +127,24 @@ export APTTMP=/ftp/tmp/apt # Set this if only the required (and NOT the recommended/suggested) packages # should be added on CDs when a package is added on the CD. #export NORECOMMENDS=1 + +# Produce jigdo files: +# 0/unset = Don't do jigdo at all, produce only the full iso image. +# 1 = Produce both the iso image and jigdo stuff. +# 2 = Produce ONLY jigdo stuff by piping mkisofs directly into jigdo-file, +# no temporary iso image is created (saves lots of disk space). +# NOTE: The no-temp-iso will not work for (at least) alpha and powerpc +# since they need the actual .iso to make it bootable. For these archs, +# the temp-iso will be generated, but deleted again immediately after the +# jigdo stuff is made; needs temporary space as big as the biggest image. +#export DOJIGDO=2 +# +# jigdo-file command & options +# Note: building the cache takes hours, so keep it around for the next run +#export JIGDOCMD="/usr/local/bin/jigdo-file --cache=$HOME/jigdo-cache.db" +# +# HTTP/FTP URL for directory where you intend to make the templates available. +# %ARCH%, if present, will be replaced by $ARCH (or "source"). This only goes +# in the .jigdo files, which you can edit easily if you want. +# No trailing slash. +#export JIGDOTEMPLATEURL="http://this-guy-didnt-configure-debiancd-correctly.com/debian-cd/templates/3.0BETA/%ARCH%" diff --git a/Makefile b/Makefile index 7862b51e..663865b1 100755 --- a/Makefile +++ b/Makefile @@ -61,6 +61,9 @@ endif ifndef BOOTDISKS export BOOTDISKS=$(MIRROR)/dists/$(CODENAME)/main/disks-$(ARCH) endif +ifndef DOJIGDO +export DOJIGDO=0 +endif ## Internal variables apt=$(BASEDIR)/tools/apt-selection @@ -142,6 +145,17 @@ ifndef NONUS ok=false endif endif +# If we do jigdo, we need a command and a URL. If not, make sure we won't. +ifneq "$(DOJIGDO)" "0" +ifndef JIGDOCMD +ok=false +endif +ifndef JIGDOTEMPLATEURL +ok=false +endif +else +export JIGDOCMD=false +endif default: @echo "Please refer to the README file for more information" @@ -688,40 +702,151 @@ $(SDIR)/CD1/dists/$(CODENAME)-secured: $(add_secured); \ done +# Make file list for jigdo (if DOJIGDO>0) +# "Fake" depend on the unstable Packages.gz to make sure we only regenerate +# this list when really necessary (saves many minutes per run). +# Don't depend on anything else as this will not work as intended, so +# make $(TDIR) ourselves just to be sure. +$(TDIR)/jigdofilelist: $(MIRROR)/dists/unstable/main/binary-i386/Packages.gz + @echo "Generating file list for jigdo (if requested) ..." + $(Q)set -e; \ + if [ "$(DOJIGDO)" != 0 ]; then \ + mkdir -p $(TDIR); \ + find $(MIRROR)//dists $(MIRROR)//doc $(MIRROR)//indices \ + $(MIRROR)//pool $(MIRROR)//project $(MIRROR)//tools \ + -type f \ + | egrep -v '/README|INDEX$$|/Maintainers|/Release$$|/debian-keyring\.tar\.gz$$|/ls-lR|//doc/[^/]+/?[^/]*\.(txt|html)$$' \ + > $(TDIR)/jigdofilelist; \ + if [ -n "$(NONUS)" ]; then \ + find $(NONUS)// -type f \ + | egrep -v '/README|INDEX$$|/Maintainers|/Release$$|/debian-keyring\.tar\.gz$$|/ls-lR|//doc/[^/]+/?[^/]*\.(txt|html)$$' \ + >> $(TDIR)/jigdofilelist; \ + fi; \ + fi # Generates all the images images: bin-images src-images -bin-images: ok bin-md5list $(OUT) + +# DOJIGDO postboot actions (source has the appropriate subset) +# 0 no isofile +# 0 yes isofile post +# 1 no isofile jigdo jigdoadd +# 1 yes isofile post jigdo jigdoadd +# 2 no isojigdo jigdoadd +# 2 yes isofile post jigdo jigdoadd rmiso +# +bin-images: ok bin-md5list $(OUT) $(TDIR)/jigdofilelist @echo "Generating the binary iso images ..." $(Q)set -e; \ for file in $(BDIR)/*.packages; do \ dir=$${file%%.packages}; \ n=$${dir##$(BDIR)/}; \ + num=$$n; \ dir=$(BDIR)/CD$$n; \ cd $$dir/..; \ opts=`cat $(BDIR)/$$n.mkisofs_opts`; \ volid=`cat $(BDIR)/$$n.volid`; \ rm -f $(OUT)/$(CODENAME)-$(ARCH)-$$n.raw; \ - $(MKISOFS) $(MKISOFS_OPTS) -V "$$volid" \ - -o $(OUT)/$(CODENAME)-$(ARCH)-$$n.raw $$opts CD$$n ; \ - if [ -f $(BASEDIR)/tools/boot/$(CODENAME)/post-boot-$(ARCH) ]; then \ - $(BASEDIR)/tools/boot/$(CODENAME)/post-boot-$(ARCH) $$n $$dir \ - $(OUT)/$(CODENAME)-$(ARCH)-$$n.raw; \ - fi \ + if [ "$(DOJIGDO)" != "2" -o -f $(BASEDIR)/tools/boot/$(CODENAME)/post-boot-$(ARCH) ]; then \ + $(MKISOFS) $(MKISOFS_OPTS) -V "$$volid" \ + -o $(OUT)/$(CODENAME)-$(ARCH)-$$n.raw $$opts CD$$n ; \ + if [ -f $(BASEDIR)/tools/boot/$(CODENAME)/post-boot-$(ARCH) ]; then \ + $(BASEDIR)/tools/boot/$(CODENAME)/post-boot-$(ARCH) $$n $$dir \ + $(OUT)/$(CODENAME)-$(ARCH)-$$n.raw; \ + fi; \ + if [ "$(DOJIGDO)" != "0" ]; then \ + $(JIGDOCMD) make-template --force \ + --files-from=$(TDIR)/jigdofilelist \ + --image=$(OUT)/$(CODENAME)-$(ARCH)-$$n.raw \ + --jigdo=$(OUT)/$(CODENAME)-$(ARCH)-$$n.jigdo \ + --template=$(OUT)/$(CODENAME)-$(ARCH)-$$n.template \ + --label Non-US="$(NONUS)" \ + --label Debian="$(MIRROR)" \ + --no-image-section --no-servers-section \ + --report=noprogress; \ + fi; \ + else \ + $(MKISOFS) $(MKISOFS_OPTS) -V "$$volid" \ + $$opts CD$$n \ + | $(JIGDOCMD) make-template --force \ + --files-from=$(TDIR)/jigdofilelist \ + --image=- \ + --jigdo=$(OUT)/$(CODENAME)-$(ARCH)-$$n.jigdo \ + --template=$(OUT)/$(CODENAME)-$(ARCH)-$$n.template \ + --label Non-US="$(NONUS)" \ + --label Debian="$(MIRROR)" \ + --no-image-section --no-servers-section \ + --report=noprogress; \ + fi; \ + if [ "$(DOJIGDO)" != "0" ]; then \ + echo "" >> $(OUT)/$(CODENAME)-$(ARCH)-$$n.jigdo; \ + echo "[Image]" \ + >> $(OUT)/$(CODENAME)-$(ARCH)-$$n.jigdo; \ + echo "Filename=debian-`echo $(DEBVERSION) | sed -e 's/[. ]//g'`-$(ARCH)-binary-$$n.iso" \ + >> $(OUT)/$(CODENAME)-$(ARCH)-$$n.jigdo; \ + echo "Template=`echo "$(JIGDOTEMPLATEURL)" | sed -e 's|%ARCH%|$(ARCH)|g'`/$(CODENAME)-$(ARCH)-$$n.template" \ + >> $(OUT)/$(CODENAME)-$(ARCH)-$$n.jigdo; \ + echo "ShortInfo='"$(BINDISKINFOND)" CD'" \ + >> $(OUT)/$(CODENAME)-$(ARCH)-$$n.jigdo; \ + echo "Info='Generated on `date -R`'" \ + >> $(OUT)/$(CODENAME)-$(ARCH)-$$n.jigdo; \ + fi; \ + if [ "$(DOJIGDO)" = "2" ]; then \ + rm -f $(OUT)/$(CODENAME)-$(ARCH)-$$n.raw; \ + fi; \ done -src-images: ok src-md5list $(OUT) +src-images: ok src-md5list $(OUT) $(TDIR)/jigdofilelist @echo "Generating the source iso images ..." $(Q)set -e; \ for file in $(SDIR)/*.sources; do \ dir=$${file%%.sources}; \ n=$${dir##$(SDIR)/}; \ + num=$$n; \ dir=$(SDIR)/CD$$n; \ cd $$dir/..; \ opts=`cat $(SDIR)/$$n.mkisofs_opts`; \ volid=`cat $(SDIR)/$$n.volid`; \ rm -f $(OUT)/$(CODENAME)-src-$$n.raw; \ - $(MKISOFS) $(MKISOFS_OPTS) -V "$$volid" \ - -o $(OUT)/$(CODENAME)-src-$$n.raw $$opts CD$$n ; \ + if [ "$(DOJIGDO)" != "2" ]; then \ + $(MKISOFS) $(MKISOFS_OPTS) -V "$$volid" \ + -o $(OUT)/$(CODENAME)-src-$$n.raw $$opts CD$$n ; \ + if [ "$(DOJIGDO)" != "0" ]; then \ + $(JIGDOCMD) make-template --force \ + --files-from=$(TDIR)/jigdofilelist \ + --image=$(OUT)/$(CODENAME)-src-$$n.raw \ + --jigdo=$(OUT)/$(CODENAME)-src-$$n.jigdo \ + --template=$(OUT)/$(CODENAME)-src-$$n.template \ + --label Non-US="$(NONUS)" \ + --label Debian="$(MIRROR)" \ + --no-image-section --no-servers-section \ + --report=noprogress; \ + fi; \ + else \ + $(MKISOFS) $(MKISOFS_OPTS) -V "$$volid" \ + $$opts CD$$n \ + | $(JIGDOCMD) make-template --force \ + --files-from=$(TDIR)/jigdofilelist \ + --image=- \ + --jigdo=$(OUT)/$(CODENAME)-src-$$n.jigdo \ + --template=$(OUT)/$(CODENAME)-src-$$n.template \ + --label Non-US="$(NONUS)" \ + --label Debian="$(MIRROR)" \ + --no-image-section --no-servers-section \ + --report=noprogress; \ + fi; \ + if [ "$(DOJIGDO)" != "0" ]; then \ + echo "" >> $(OUT)/$(CODENAME)-src-$$n.jigdo; \ + echo "[Image]" \ + >> $(OUT)/$(CODENAME)-src-$$n.jigdo; \ + echo "Filename=debian-`echo $(DEBVERSION) | sed -e 's/[. ]//g'`-source-$$n.iso" \ + >> $(OUT)/$(CODENAME)-src-$$n.jigdo; \ + echo "Template=`echo "$(JIGDOTEMPLATEURL)" | sed -e 's|%ARCH%|source|g'`/$(CODENAME)-src-$$n.template" \ + >> $(OUT)/$(CODENAME)-src-$$n.jigdo; \ + echo "ShortInfo='"$(SRCDISKINFOND)" CD'" \ + >> $(OUT)/$(CODENAME)-src-$$n.jigdo; \ + echo "Info='Generated on `date -R`'" \ + >> $(OUT)/$(CODENAME)-src-$$n.jigdo; \ + fi; \ done # Generate the *.list files for the Pseudo Image Kit @@ -754,11 +879,23 @@ src-image: ok src-md5list $(OUT) -o $(OUT)/$(CODENAME)-src-$(CD).raw $$opts CD$(CD) -#Calculate the md5sums for the images +#Calculate the md5sums for the images (if available), or get from templates imagesums: $(Q)cd $(OUT); :> MD5SUMS; for file in `find * -name \*.raw`; do \ - md5sum $$file >>MD5SUMS; \ - done + md5sum $$file >>MD5SUMS; \ + done; \ + for file in `find * -name \*.template`; do \ + if [ "`tail --bytes=29 "$$file" | head --bytes=1 | od -tx1 -An | sed -e 's/ //g'`" != 01 ]; then \ + echo "Possibly invalid template $$file"; exit 1; \ + fi; \ + grep -q " $${file%%.template}.raw"'$$' MD5SUMS \ + || echo "`tail --bytes=22 "$$file" | head --bytes=16 | od -tx1 -An | sed -e 's/ //g'` $${file%%.template}.raw" >>MD5SUMS; \ + done + +# Likewise, the file size can be extracted from the .template with: +# tail --bytes=28 $$file | head --bytes=6 | od -tx1 -An \ +# | tr ' abcdef' '\nABCDEF' | tac | tr '\n' ' ' \ +# | sed -e 's/ //g; s/^.*$/ibase=16 & /' | tr ' ' '\n' | bc ## MISC TARGETS ## diff --git a/README b/README index 01700c21..31433766 100644 --- a/README +++ b/README @@ -26,6 +26,7 @@ Software : - lynx (for text version of README.html) - if you want to generate .list files: tempfile, as included in debianutils >= 1.6 (not absolutely necessary, workaround described in tools/pi-makelist) +- if you want to generate jigdo files: jigdo-file (see below) Other : - lots of free space on your disks - a Debian mirror (on a partition where you can write, if you can't write @@ -253,6 +254,33 @@ sources from http://www.fokus.gmd.de/research/cc/glone/employees/joerg.schilling Alternatively (and certainly simpler for people using Debian system), the isoinfo binary is in the mkisofs package (since Debian potato at least). +Jigdo +----- +People wanting to offer the CD images in jigdo format (Jigsaw Download, +the successor to the Pseudo-Image Kit), can indicate so by setting the +DOJIGDO and related variables in CONF.sh. So this is not a target itself, +but a modification of the "images" targets. You can choose only iso +generation (default), only jigdo generation (for highly reduced disk +usage), or both iso and jigdo generation. + +Information on jigdo is available at http://www.debian.org/CD/jigdo-cd/ + +To generate the jigdo files and templates, the "jigdo-file" program is +needed, which is available via that same URL. + +Jigdo-file creates and maintains a cache/database file with checksums of +all files on your Debian mirror. The first time, this may take hours to be +generated (use "top" to see what's going on), so it's wise to keep the +cache in your homedir and not delete it ever. + +Note that jigdo-file can easily use 60+ MB of working memory, so don't use +this on machines with less than 128 MB RAM. + +The MD5SUMS file generated by the "imagesums" target will contain the MD5 +checksums of all generated images, regardless of the DOJIGDO setting. If +no full iso image is available, the MD5sum will be extracted from the +.template file. A note in the Makefile shows how the original file size +can be extracted from the .template in a similar way. About the hook system ===================== diff --git a/debian/CONF.sh b/debian/CONF.sh index 921bd7a2..0b86524c 100644 --- a/debian/CONF.sh +++ b/debian/CONF.sh @@ -14,6 +14,10 @@ unset MKISOFS_OPTS unset EXCLUDE unset SRCEXCLUDE unset NORECOMMENDS +unset DOJIGDO +unset JIGDOCMD +unset JIGDOTEMPLATEURL + # The debian-cd dir export BASEDIR=/usr/share/debian-cd @@ -118,3 +122,24 @@ export APTTMP=/home/ftp/tmp/apt # Set this if only the required (and NOT the recommended/suggested) packages # should be added on CDs when a package is added on the CD. #export NORECOMMENDS=1 + +# Produce jigdo files: +# 0/unset = Don't do jigdo at all, produce only the full iso image. +# 1 = Produce both the iso image and jigdo stuff. +# 2 = Produce ONLY jigdo stuff by piping mkisofs directly into jigdo-file, +# no temporary iso image is created (saves lots of disk space). +# NOTE: The no-temp-iso will not work for (at least) alpha and powerpc +# since they need the actual .iso to make it bootable. For these archs, +# the temp-iso will be generated, but deleted again immediately after the +# jigdo stuff is made; needs temporary space as big as the biggest image. +#export DOJIGDO=2 +# +# jigdo-file command & options +# Note: building the cache takes hours, so keep it around for the next run +#export JIGDOCMD="/usr/local/bin/jigdo-file --cache=$HOME/jigdo-cache.db" +# +# HTTP/FTP URL for directory where you intend to make the templates available. +# %ARCH%, if present, will be replaced by $ARCH (or "source"). This only goes +# in the .jigdo files, which you can edit easily if you want. +# No trailing slash. +#export JIGDOTEMPLATEURL="http://this-guy-didnt-configure-debiancd-correctly.com/debian-cd/templates/3.0BETA/%ARCH%" diff --git a/debian/changelog b/debian/changelog index 9ad1e8b0..dfd6c4ed 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,7 @@ debian-cd (2.2.13) unstable; urgency=low * UNRELEASED. + * Add jigdo generation support (see README). -- Raphael Hertzog Sun, 20 Jan 2002 15:42:29 +0100