diff --git a/CONF.sh b/CONF.sh index 7e33050b..4b2b0725 100644 --- a/CONF.sh +++ b/CONF.sh @@ -3,14 +3,11 @@ # # Unset all optional variables first to start from a clean state -unset NONUS || true -unset FORCENONUSONCD1 || true unset NONFREE || true unset CONTRIB || true unset EXTRANONFREE || true unset LOCAL || true unset LOCALDEBS || true -unset SECURED || true unset SECURITY || true unset BOOTDIR || true unset SYMLINK || true @@ -29,11 +26,7 @@ unset JIGDOFALLBACKURLS || true unset JIGDOINCLUDEURLS || true unset JIGDOSCRIPT || true unset JIGDO_OPTS || true -unset DEFBINSIZE || true -unset DEFSRCSIZE || true -unset FASTSUMS || true unset PUBLISH_URL || true -unset PUBLISH_NONUS_URL || true unset PUBLISH_PATH || true unset UDEB_INCLUDE || true unset UDEB_EXCLUDE || true @@ -99,14 +92,6 @@ fi # Paths to the mirrors export MIRROR=/mirror/debian -# Comment the following line if you don't have/want non-US -#export NONUS=/ftp/debian-non-US - -# And this option will make you 2 copies of CD1 - one with all the -# non-US packages on it, one with none. Useful if you're likely to -# need both. -#export FORCENONUSONCD1=1 - # Path of the temporary directory export TDIR=/mirror/tmp @@ -137,11 +122,6 @@ export CONTRIB=1 # containing dists/$CODENAME/local/binary-$ARCH # export LOCALDEBS=/home/joey/debian/va/debian -# If you want a -secured tree with a copy of the signed -# Release.gpg and files listed by this Release file, then -# uncomment this line -# export SECURED=1 - # Where to find the security patches. This directory should be the # top directory of a security.debian.org mirror. #export SECURITY="$TOPDIR"/debian/debian-security @@ -166,7 +146,7 @@ export CONTRIB=1 export ISOLINUX=1 # uncomment this to if you want to see more of what the Makefile is doing -export VERBOSE_MAKE=1 +#export VERBOSE_MAKE=1 # uncoment this to make build_all.sh try to build a simple CD image if # the proper official CD run does not work @@ -180,7 +160,7 @@ ATTEMPT_FALLBACK=yes # CD700: (semi-)standard 80-min CD (700 MiB) # DVD: standard 4.7 GB DVD # CUSTOM: up to you - specify a size to go with it (in 2K blocks) -export DISKTYPE=DVD +export DISKTYPE=CD #export DISKTYPE=CUSTOM #export CUSTOMSIZE=XXXX @@ -213,10 +193,6 @@ export NORECOMMENDS=1 # jigdo stuff is made; needs temporary space as big as the biggest image. export DOJIGDO=1 -# jigdo-file command & options -# Note: building the cache takes hours, so keep it around for the next run -#export JIGDOCMD="/usr/local/bin/jigdo-file --cache=$HOME/jigdo-cache.db" -# # HTTP/FTP URL for directory where you intend to make the templates # available. You should not need to change this; the default value "" # means "template in same dir as the .jigdo file", which is usually @@ -251,15 +227,10 @@ export JIGDOINCLUDEURLS="http://cdimage.debian.org/debian-cd/debian-servers.jigd # data. #export JIGDOSCRIPT="myscript" -# If set, use the md5sums from the main archive, rather than calculating -# them locally -export FASTSUMS=1 - # A couple of things used only by publish_cds, so it can tweak the # jigdo files, and knows where to put the results. # You need to run publish_cds manually, it is not run by the Makefile. export PUBLISH_URL="http://cdimage.debian.org/jigdo-area" -export PUBLISH_NONUS_URL="http://non-US.cdimage.debian.org/jigdo-area" export PUBLISH_PATH="/home/jigdo-area/" # Specify files and directories to *exclude* from jigdo processing. These @@ -315,7 +286,7 @@ done #export KERNEL_PARAMS="DEBCONF_PRIORITY=critical" # If set, limits the number of binary CDs to produce. -# export MAXCDS=1 +export MAXCDS=1 # If set, overrides the boot picture used. #export SPLASHPNG="$BASEDIR/data/$CODENAME/splash-img.png" @@ -328,7 +299,7 @@ done # If so we will link to them on the web site. export OMIT_RELEASE_NOTES=1 -# Set this to override the defaul location +# Set this to override the default location #export RELEASE_NOTES_LOCATION="http://www.debian.org/releases/$CODENAME" case "$OFFICIAL" in @@ -342,3 +313,51 @@ case "$OFFICIAL" in export OFFICIAL_VAL=0 ;; esac + +################################## +# LOCAL HOOK DEFINITIONS +################################## +# +# Set these to point to scripts/programs to be called at various +# points in the debian-cd image-making process. This is the ideal place +# to customise what's on the CDs, for example to add extra files or +# modify existing ones. Each will be called with the arguments in order: +# +# $TDIR (the temporary dir containing the build tree) +# $MIRROR (the location of the mirror) +# $DISKNUM (the image number in the set) +# $CDDIR (the root of the temp disc tree) +# $ARCHES (the set of architectures chosen) +# +# BE CAREFUL about what you do at each point: in the first couple of +# cases, files and directories you're looking to use may not exist yet, +# you may need to worry about adding entries into md5sum.txt yourself +# and (in the last couple of cases) if you add any extra files you may +# end up over-filling the disc. If you *do* need to add files at the end +# of the process, see RESERVED_BLOCKS_HOOK below. It's strongly +# recommended to do this kind of customisation up-front if you can, it's +# much simpler that way! + +# The disc_start hook. This will be called near the beginning of the +# start_new_disc script, just after the directory tree has been created +# but before any files have been added +#export DISC_START_HOOK=/bin/true + +# The disc_pkg hook. This will be called just after the +# start_new_disc script has finished, just before make_disc_trees.pl +# starts to add package files. +#export DISC_PKG_HOOK=/bin/true + +# The reserved_blocks hook; if set, this script should print the +# number of 2K blocks that need to be reserved for data to be added +# *after* a disc tree is filled with packages. +#export RESERVED_BLOCKS_HOOK=/bin/true + +# The disc_finish hook. This will be called once a disc image is full, +# just after the last package rollback but before the last bits of +# cleanup are done on the temp disc tree +#export DISC_FINISH_HOOK=/bin/true + +# The disc_end hook. This will be called *right* at the end of the +# image-making process in make_disc_trees.pl. +#export DISC_END_HOOK=/bin/true diff --git a/Makefile b/Makefile index 19cd8fe8..27db95df 100755 --- a/Makefile +++ b/Makefile @@ -51,14 +51,9 @@ endif ## Internal variables apt=$(BASEDIR)/tools/apt-selection list2cds=$(BASEDIR)/tools/list2cds -scansources=$(BASEDIR)/tools/scansources -add_files=$(BASEDIR)/tools/add_files -add_secured=$(BASEDIR)/tools/add_secured md5sum=md5sum -fastsums=$(BASEDIR)/tools/fast_sums jigdo_cleanup=$(BASEDIR)/tools/jigdo_cleanup grab_md5=$(BASEDIR)/tools/grab_md5 -dedicated-src=$(BASEDIR)/tools/dedicated_source make_image=$(BASEDIR)/tools/make_image merge_package_lists=$(BASEDIR)/tools/merge_package_lists update_popcon=$(BASEDIR)/tools/update_popcon @@ -165,7 +160,7 @@ clean: ok dir-clean dir-clean: $(Q)rm -rf $(BDIR)/CD[1234567890]* $(Q)rm -f $(BDIR)/*.filelist* - $(Q)rm -f $(BDIR)/packages-stamp $(BDIR)/upgrade-stamp $(BDIR)/secured-stamp $(BDIR)/md5-check + $(Q)rm -f $(BDIR)/packages-stamp $(BDIR)/upgrade-stamp $(BDIR)/md5-check # Completely cleans the current arch tree realclean: distclean @@ -370,47 +365,8 @@ need-complete-mirror: exit 1; \ fi -dedicated-src: ok -# false - -## EXTRAS ## - -# Launch the extras scripts correctly for customizing the CDs -extras: bin-extras -bin-extras: ok - $(Q)if [ -z "$(DIR)" -o -z "$(CD)" -o -z "$(ROOTSRC)" ]; then \ - echo "Give me more parameters (DIR, CD and ROOTSRC are required)."; \ - false; \ - fi - @echo "Adding dirs '$(DIR)' from '$(ROOTSRC)' to '$(BDIR)/CD$(CD)'" ... - $(Q)$(add_files) $(BDIR)/CD$(CD) $(ROOTSRC) $(DIR) -src-extras: - $(Q)if [ -z "$(DIR)" -o -z "$(CD)" -o -z "$(ROOTSRC)" ]; then \ - echo "Give me more parameters (DIR, CD and ROOTSRC are required)."; \ - false; \ - fi - @echo "Adding dirs '$(DIR)' from '$(ROOTSRC)' to '$(BDIR)/CD$(CD)'" ... - $(Q)$(add_files) $(BDIR)/CD$(CD) $(ROOTSRC) $(DIR) - ## IMAGE BUILDING ## -# Generate $CODENAME-secured tree with Packages and Release(.gpg) files -# from the official tree -# Complete the Release file from the normal tree -secured: bin-secured #src-secured -bin-secured: $(BDIR)/secured-stamp -$(BDIR)/secured-stamp: - @echo "Generating $(CODENAME)-secured on all the binary CDs ..." - $(Q)set -e; \ - for file in $(BDIR)/*.packages; do \ - dir=$${file%%.packages}; \ - n=$${dir##$(BDIR)/}; \ - dir=$(BDIR)/CD$$n; \ - cd $$dir; \ - $(add_secured); \ - done - $(Q)touch $(BDIR)/secured-stamp - # DOJIGDO actions (for both binaries and source) # 0 isofile # 1 isofile + jigdo, cleanup_jigdo @@ -423,7 +379,7 @@ check-number-given: @test -n "$(CD)" || (echo "Give me a CD= parameter !" && false) # Generate only one image number $(CD) -image: check-number-given image +image: check-number-given images # Calculate the md5sums for the images (if available), or get from templates imagesums: diff --git a/README b/README index e207c47c..7ca3fa3d 100644 --- a/README +++ b/README @@ -1,16 +1,16 @@ -YACS - Yet Another CD Script :-) -================================ -(better known as debian-cd) +debian-cd +========= -Copyright 1999-2001 Raphaël Hertzog -This set of tools is licensed under the General Public License -version 2 or any later version. You can find it in +Copyright 1999-2001 Raphaël Hertzog and others, +2004-2006 Steve McIntyre +This set of tools is licensed under the General Public License version +2 or any later version. You can find it in /usr/share/common-licenses/GPL on a Debian GNU system. -Thanks to Steve McIntyre for his -work on slink_cd/debian_cd. Some ideas come from his script. +Some of the ideas here (a loooong time ago) came from Steve McIntyre's +slink_cd script. -Thanks to all the contributors on the debian-cd mailing list. +Thanks also to all the contributors on the debian-cd mailing list. What is needed? =============== @@ -20,9 +20,10 @@ Software: - perl (>= 5.004) - bash (or another POSIX shell) - make -- mkisofs/mkhybrid (mkisofs also provides the isoinfo binary used by the - Pseudo Image Kit) +- cpp +- mkisofs/genisoimage - the perl Digest::MD5 module +- the perl Compress::Zlib module - lynx (for text version of README.html) and todos from sysutils to convert docs to DOS format (although you can rip that out, too) - if you want to generate jigdo files: jigdo-file (see below) @@ -46,10 +47,7 @@ $ sensible-editor CONF.sh $ . CONF.sh $ make distclean $ make status -$ make list COMPLETE=1 SIZELIMIT1=625000000 SRCSIZELIMIT=665000000 $ make official_images -[ alternatively, if you only want binary images: - $ make bin-official_images ] However, you really should consider reading further for more information. You can also take a look at build.sh and build_all.sh for an automatized @@ -103,33 +101,25 @@ The process of building a CD is composed of the following steps: - now you can decide what you want on your CDs - $ make list TASK=tasks/debian-2.2 COMPLETE=1 + $ make packagelists TASK=tasks/debian-2.2 COMPLETE=1 or - $ make list TASK=tasks/gnome COMPLETE=0 SIZELIMIT=576716800 + $ make packagelists TASK=tasks/gnome COMPLETE=0 or - $ export NONFREE=1; make list TASK=tasks/your-task-here COMPLETE=1 + $ export NONFREE=1; make packagelists TASK=tasks/your-task-here COMPLETE=1 or for something like an official image for the USA (without non-US & non-free) : - $ make list COMPLETE=1 SIZELIMIT1=576716800 + $ make packagelists COMPLETE=1 .... take a look at the file tasks/* to see the options you can have :) You can change the behaviour of this command with the following variables: - - if NONUS is set, then packages from non-US will be allowed (the value - of NONUS must be the path to the non-US mirror if you have one) - - if FORCENONUSONCD1 is set also, then packages will be - rearranged so that non-US packages are all on CD#1. This - includes the non-free ones if you specify NONFREE. Then 2 separate - copies of CD#1 will be produced, identical in every respect except - for the inclusion/lack of non-US packages. The same happens for - the source CDs when they are built. - if NONFREE is set, then packages from non-free will be allowed (NONFREE must be exported to all sub-shells) - if EXTRANONFREE is set, then non-free packages will be included @@ -138,23 +128,30 @@ The process of building a CD is composed of the following steps: (EXTRANONFREE must be exported to all sub-shells) - if COMPLETE is set, all packages that are not listed in the selected task file will be included at the end - - if SIZELIMIT is set, it will be used as the maximum size that - we can put into each CD - - if SIZELIMIT (with beeing a integer) is set, it will be used - as the maximum size of the X'th binary CD. SIZELIMIT overrides - SIZELIMIT ... - - if SRCSIZELIMIT is set, it's used as the maximum size for source CD - - if SRCSIZELIMIT (with beeing an integer) is set, it will be - used as the maximum size of the X'th source CD + - setting INSTALLER_CD will use an appropriate task file for + building small CDs (businesscard and netinst) - This target calls the targets "bin-list" and "src-list" that can be used to - build only binary CDs or only source CDs. +- now, we'll start making temporary trees: -- it may be time to add the disks-arch stuff and to make the CDs bootable: + $ make image-trees - $ make bootable + This will first work sort the list of packages for each architecture + into order so that standard, required, important and base packages + are placed first, then other packages will be added in the order + given modulo dependency ordering. Once the sorted list is created, + the different architecture lists will be merged (if more than one + architecture is selected). - This affects only the binary CDs. + Then the code will start laying out temporary directory trees for + the CDs. In order, this includes the following steps: + + - Creating an empty directory layout + - Generating an image label and volume ID and other metadata such as + debian-installer information files + - Add documentation and installation/upgrade tools + - Add Release files and other archive metadata + - Make the image bootable for the selected architecture(s) + - Start generating the md5sum.txt file If you want to use boot-floppies built by yourself you can add a parameter BOOTDISKS= which specifies the directory where @@ -163,57 +160,40 @@ The process of building a CD is composed of the following steps: mirror. Your boot-floppies must also be on the same partition as your mirror and temporary dir (hardlinks are used here too). - Size calculation is imprecise for bootable CDs (usually the first CDs - in the set), so you may need to manually adjust SIZELIMIT* variables - to account for the size of the booting stuff used. + Once the disc tree has all of this start data, we start filling the + directory trees with packages from the sorted list. The size of the + image to be created is set using DISKTYPE in CONF.sh; if the + standard sizes do not match what you're after, use DISKTYPE=CUSTOM + and specify your own size using CUSTOMSIZE. The algorithm is simple + for adding packages: -- now, we'll add the binary packages to the temporary tree: + - link the package into the temporary disc tree + - append the metadata to the appropriate Packages or Sources file + - add md5sum information for the added file(s) to the md5sum.txt file - $ make packages + This continues until the temporary tree grows one package *too + large* for the selected image size. At that point, we roll back the + last set of changes associated with that package. Then: -- and we'll add the sources to the temporary tree: + - check if the disc contains all the packages needed to install a base system + - finish off the Release file, using the checksums of the + Packages/Sources files we generated + - finish off the md5sum.txt file - $ make sources - -- if you want to install additional files: - - $ make bin-extras CD=1 ROOTSRC=/home/ftp/ DIR=goodies/wordperfect - $ make src-extras CD=3 ROOTSRC=/home/ftp/ DIR=goodies/kernel-2.3 - - The first will copy /home/ftp/goodies/wordperfect/ to the first binary - CD. It will be in /goodies/wordperfect/. You can call - make extras multiple times if you need more. - - Please note that the files to be copied should be on the same partition - than your mirror (unless you use a symlink farm). - - If you want to do customize your CD even more, you can use the hook - system. Read more about that below. - -- We can add an md5sum.txt file on each CD to enable users to check their - files: - - $ make md5list - - This calls the targets 'bin-md5list' and 'src-md5list'. You can - choose to call only bin-md5list if you're building only binary images. + Next, we continue to the next disc tree, using the same process: + start it, copy packages in until they overflow, roll back and + finish. And repeat. Each time a package is found to be too large to + fit inside an image, it will be kept back and will (obviously) be + the first package placed into the next disc tree. - now we can create the images: $ make images If you don't have enough space for all images, you can generate - only one image (of the second binary CD for example) with: + only one image (of the second CD for example) with: - $ make bin-image CD=2 - - Of course if you want to build all binary images you'll use: - - $ make bin-images - - The following will generate the source images: - - $ make src-images + $ make image CD=2 Note: here we use "make images", but you could as well use "make official_images" since the latter is the same as the former @@ -225,7 +205,6 @@ The process of building a CD is composed of the following steps: $ make imagesums - Official images =============== @@ -281,16 +260,9 @@ for jigdo, merely a modification of the "images" targets. You can choose only .iso generation (default), only .jigdo generation (for highly reduced disk usage), or both .iso and .jigdo generation. -To generate the jigdo files and templates, you need the "jigdo-file" program, -which is available via the same URL. - -Jigdo-file creates and maintains a cache/database file with checksums of -all files on your Debian mirror. The first time, this may take hours to be -generated (use "top" to see what's going on), so it's wise to keep the -cache in your homedir and not delete it ever. - -Note that jigdo-file can easily use 60+ MB of working memory, so don't use -this on machines with less than 128 MB RAM. +To generate the jigdo files and templates, you need an +appropriately-patched version of mkisofs/genisoimage, as shipped in +Debian Etch. The MD5SUMS file generated by the "imagesums" target will contain the MD5 checksums of all generated images, regardless of the DOJIGDO setting. If @@ -322,24 +294,15 @@ debian-cd allows you to automatically create a directory on disc which is suitable for use as a fallback mirror. It is populated with hard links to the archive contents. In CONF.sh, simply supply as JIGDOFALLBACKPATH the name of the directory, and as JIGDOFALLBACKURLS -the URLs under which it will be made available (two separate URLs are -necessary, one for "Debian" and one for "Non-US"). - +the URLs under which it will be made available. About the hook system ===================== -A hook script can be executed at different times. You can specify the -script by setting the HOOK variable to the script filename. It will -get 2 arguments, the first is the CD number. The second depends on -where/when the hook script is called. It can be 'before-scanpackages' -or 'before-mkisofs' (their values are explicit ...). When the script -is called, the current directory will be the temporary directory used for -the build (aka $TDIR/$CODENAME-$ARCH). - -There are hooks only for binary CDs at the present time. If HOOK is not -set, it will look for a script $BASEDIR/tools/$CODENAME.hook. - +A hook script can be executed at different times during the CD build +process to customise your CDs. You can specify the script by setting +the various HOOK variables in CONF.sh; look there for more information +about what hook points are available. About the symlink farm ====================== diff --git a/README.devel b/README.devel index bb63757b..5a02c29c 100644 --- a/README.devel +++ b/README.devel @@ -1,5 +1,5 @@ -README for those who want to hack on yacs/debian-cd ---------------------------------------------------- +README for those who want to hack on debian-cd +---------------------------------------------- Organisation : -------------- @@ -12,10 +12,9 @@ auth: svn+ssh://svn.debian.org/svn/debian-cd/trunk anon: svn://svn.debian.org/debian-cd/trunk web: http://svn.debian.org/wsvn/debian-cd -If you want to discuss anything related to the debian-cd development, you -can contact me directly or, better, you can mail the -debian-cd@lists.debian.org mailing list where all people interested in the -debian-cd development are subscribed. +If you want to discuss anything related to the debian-cd development, +mail the debian-cd@lists.debian.org mailing list where all people +interested in the debian-cd development are subscribed. Technical details : ------------------- @@ -36,8 +35,8 @@ related) like the master file from boot-floppies and so on. Each tool is self-documented, if you want to know what it does read the sources (they are scripts). -Debugging YACS : ----------------- +Debugging debian-cd : +--------------------- If you want to read more about what YACS is doing you can set the VERBOSE environment variable to 1, 2 or 3 depending on the @@ -47,16 +46,14 @@ Some scripts generates their own log files in the temp dir. You can take a look at them if you want to check for warnings and so on. Log files : -- $TDIR/$CODENAME-$ARCH/log.list2cds -- $TDIR/$CODENAME-SRC/log.cds2src +- $TDIR/$CODENAME/log.list2cds +- $TDIR/$CODENAME/make_disc_trees.log TODO list : ----------- + put the doc directory only on the first binary CD + a way to force the inclusion of packages (even broken) + possibility to add project/* to the last CD -+ check the source files on the mirror (with Sources.gz) - when doing make mirrorcheck Bugs / Problems : ----------------- @@ -75,7 +72,4 @@ The el-torito multiboot just displays a menu with each item being the same "2.88Mb boot image". Both multiboot mechanism may not work on older hardware with very old BIOSes however the el-torito one works a bit better because it's usually able to boot the first choice -without displaying the menu then. This is a non-issue since people -may still boot on CD 2,3,4,5 with the usual boot method if the multiboot -doesn't work. - +without displaying the menu then. diff --git a/debian/CONF.sh b/debian/CONF.sh index 7e33050b..4b2b0725 100644 --- a/debian/CONF.sh +++ b/debian/CONF.sh @@ -3,14 +3,11 @@ # # Unset all optional variables first to start from a clean state -unset NONUS || true -unset FORCENONUSONCD1 || true unset NONFREE || true unset CONTRIB || true unset EXTRANONFREE || true unset LOCAL || true unset LOCALDEBS || true -unset SECURED || true unset SECURITY || true unset BOOTDIR || true unset SYMLINK || true @@ -29,11 +26,7 @@ unset JIGDOFALLBACKURLS || true unset JIGDOINCLUDEURLS || true unset JIGDOSCRIPT || true unset JIGDO_OPTS || true -unset DEFBINSIZE || true -unset DEFSRCSIZE || true -unset FASTSUMS || true unset PUBLISH_URL || true -unset PUBLISH_NONUS_URL || true unset PUBLISH_PATH || true unset UDEB_INCLUDE || true unset UDEB_EXCLUDE || true @@ -99,14 +92,6 @@ fi # Paths to the mirrors export MIRROR=/mirror/debian -# Comment the following line if you don't have/want non-US -#export NONUS=/ftp/debian-non-US - -# And this option will make you 2 copies of CD1 - one with all the -# non-US packages on it, one with none. Useful if you're likely to -# need both. -#export FORCENONUSONCD1=1 - # Path of the temporary directory export TDIR=/mirror/tmp @@ -137,11 +122,6 @@ export CONTRIB=1 # containing dists/$CODENAME/local/binary-$ARCH # export LOCALDEBS=/home/joey/debian/va/debian -# If you want a -secured tree with a copy of the signed -# Release.gpg and files listed by this Release file, then -# uncomment this line -# export SECURED=1 - # Where to find the security patches. This directory should be the # top directory of a security.debian.org mirror. #export SECURITY="$TOPDIR"/debian/debian-security @@ -166,7 +146,7 @@ export CONTRIB=1 export ISOLINUX=1 # uncomment this to if you want to see more of what the Makefile is doing -export VERBOSE_MAKE=1 +#export VERBOSE_MAKE=1 # uncoment this to make build_all.sh try to build a simple CD image if # the proper official CD run does not work @@ -180,7 +160,7 @@ ATTEMPT_FALLBACK=yes # CD700: (semi-)standard 80-min CD (700 MiB) # DVD: standard 4.7 GB DVD # CUSTOM: up to you - specify a size to go with it (in 2K blocks) -export DISKTYPE=DVD +export DISKTYPE=CD #export DISKTYPE=CUSTOM #export CUSTOMSIZE=XXXX @@ -213,10 +193,6 @@ export NORECOMMENDS=1 # jigdo stuff is made; needs temporary space as big as the biggest image. export DOJIGDO=1 -# jigdo-file command & options -# Note: building the cache takes hours, so keep it around for the next run -#export JIGDOCMD="/usr/local/bin/jigdo-file --cache=$HOME/jigdo-cache.db" -# # HTTP/FTP URL for directory where you intend to make the templates # available. You should not need to change this; the default value "" # means "template in same dir as the .jigdo file", which is usually @@ -251,15 +227,10 @@ export JIGDOINCLUDEURLS="http://cdimage.debian.org/debian-cd/debian-servers.jigd # data. #export JIGDOSCRIPT="myscript" -# If set, use the md5sums from the main archive, rather than calculating -# them locally -export FASTSUMS=1 - # A couple of things used only by publish_cds, so it can tweak the # jigdo files, and knows where to put the results. # You need to run publish_cds manually, it is not run by the Makefile. export PUBLISH_URL="http://cdimage.debian.org/jigdo-area" -export PUBLISH_NONUS_URL="http://non-US.cdimage.debian.org/jigdo-area" export PUBLISH_PATH="/home/jigdo-area/" # Specify files and directories to *exclude* from jigdo processing. These @@ -315,7 +286,7 @@ done #export KERNEL_PARAMS="DEBCONF_PRIORITY=critical" # If set, limits the number of binary CDs to produce. -# export MAXCDS=1 +export MAXCDS=1 # If set, overrides the boot picture used. #export SPLASHPNG="$BASEDIR/data/$CODENAME/splash-img.png" @@ -328,7 +299,7 @@ done # If so we will link to them on the web site. export OMIT_RELEASE_NOTES=1 -# Set this to override the defaul location +# Set this to override the default location #export RELEASE_NOTES_LOCATION="http://www.debian.org/releases/$CODENAME" case "$OFFICIAL" in @@ -342,3 +313,51 @@ case "$OFFICIAL" in export OFFICIAL_VAL=0 ;; esac + +################################## +# LOCAL HOOK DEFINITIONS +################################## +# +# Set these to point to scripts/programs to be called at various +# points in the debian-cd image-making process. This is the ideal place +# to customise what's on the CDs, for example to add extra files or +# modify existing ones. Each will be called with the arguments in order: +# +# $TDIR (the temporary dir containing the build tree) +# $MIRROR (the location of the mirror) +# $DISKNUM (the image number in the set) +# $CDDIR (the root of the temp disc tree) +# $ARCHES (the set of architectures chosen) +# +# BE CAREFUL about what you do at each point: in the first couple of +# cases, files and directories you're looking to use may not exist yet, +# you may need to worry about adding entries into md5sum.txt yourself +# and (in the last couple of cases) if you add any extra files you may +# end up over-filling the disc. If you *do* need to add files at the end +# of the process, see RESERVED_BLOCKS_HOOK below. It's strongly +# recommended to do this kind of customisation up-front if you can, it's +# much simpler that way! + +# The disc_start hook. This will be called near the beginning of the +# start_new_disc script, just after the directory tree has been created +# but before any files have been added +#export DISC_START_HOOK=/bin/true + +# The disc_pkg hook. This will be called just after the +# start_new_disc script has finished, just before make_disc_trees.pl +# starts to add package files. +#export DISC_PKG_HOOK=/bin/true + +# The reserved_blocks hook; if set, this script should print the +# number of 2K blocks that need to be reserved for data to be added +# *after* a disc tree is filled with packages. +#export RESERVED_BLOCKS_HOOK=/bin/true + +# The disc_finish hook. This will be called once a disc image is full, +# just after the last package rollback but before the last bits of +# cleanup are done on the temp disc tree +#export DISC_FINISH_HOOK=/bin/true + +# The disc_end hook. This will be called *right* at the end of the +# image-making process in make_disc_trees.pl. +#export DISC_END_HOOK=/bin/true diff --git a/debian/changelog b/debian/changelog index f7128f3e..bb473590 100644 --- a/debian/changelog +++ b/debian/changelog @@ -172,6 +172,16 @@ debian-cd (3.0.0) UNRELEASED; urgency=low * CONF.sh: don't determine ARCHES if already set; don't export CPU and KERNEL. + [ Steve McIntyre ] + * Update documentation to at least vaguely match how things work. + * Added support for hook scripts in a few places around the CD + creation process, to help people customise their CDs. Quite a + lot of refactoring needed in make_disc_trees.pl needed to do that. + * Removed more cruft: + + old defs in CONF.sh + + old defs and rules in the Makefile + + tools/add_secured + -- Frans Pop Mon, 18 Dec 2006 16:20:22 +0100 debian-cd (2.2.25) unstable; urgency=low diff --git a/tools/add_secured b/tools/add_secured deleted file mode 100755 index 18f09996..00000000 --- a/tools/add_secured +++ /dev/null @@ -1,94 +0,0 @@ -#!/bin/sh - -# We are in the temp directory of a CD image -# aka $[SB]DIR/CD/ -# Everything is already in place (just md5sum.txt missing) - - -# Complete the "Release" file if it exists - -if [ -e "dists/$CODENAME/Release" ]; then - # Add the MD5Sum field again - echo "MD5Sum:" >> dists/$CODENAME/Release - find dists/$CODENAME/ -name 'Packages' -o -name 'Packages.gz' \ - -o -name 'Sources' -o -name 'Sources.gz' -o -name 'Release' | \ - grep -v non-US/ | grep -v dists/$CODENAME/Release | \ - sed -e "s#^dists/$CODENAME/##" | \ - (while read file; do \ - rfile="dists/$CODENAME/$file"; \ - c=`wc -c < $rfile`; \ - m=`md5sum < $rfile | cut -d" " -f1`; \ - printf " %s %8d %s\n" $m $c $file; \ - done) >> dists/$CODENAME/Release - # TODO: Should add SHA1 field here - # Check with libdigest-sha1-perl -fi - -if [ -n "$NONUS" -a -e "dists/$CODENAME/non-US/Release" ]; then - # Add the MD5Sum field again - echo "MD5Sum:" >> dists/$CODENAME/non-US/Release - find dists/$CODENAME/non-US/ -name 'Packages' -o -name 'Packages.gz' \ - -o -name 'Sources' -o -name 'Sources.gz' -o -name 'Release' | \ - grep -v dists/$CODENAME/non-US/Release | \ - sed -e "s#^dists/$CODENAME/non-US/##" | \ - (while read file; do \ - rfile="dists/$CODENAME/non-US/$file"; \ - c=`wc -c < $rfile`; \ - m=`md5sum < $rfile | cut -d" " -f1`; \ - printf " %s %8d %s\n" $m $c $file; \ - done) >> dists/$CODENAME/non-US/Release - # TODO: Should add SHA1 field here - # Check with libdigest-sha1-perl -fi - -# Create the $CODENAME-secured tree by copying files from the mirror -# And including the Release file and its signature - -if [ -n "$SECURED" ]; then - -test -d "dists/$CODENAME-secured" || mkdir dists/$CODENAME-secured -if [ -e "$MIRROR/dists/$CODENAME/Release" ]; then - # Copy Release and Release.gpg files - cp $MIRROR/dists/$CODENAME/Release dists/$CODENAME-secured/ - if [ -e "$MIRROR/dists/$CODENAME/Release.gpg" ]; then - cp $MIRROR/dists/$CODENAME/Release.gpg dists/$CODENAME-secured/ - fi - # Create the .aptignr file in the standard tree - :> dists/$CODENAME/.aptignr - # Include files listed in Release that corresponds to our arch - grep "^ " dists/$CODENAME-secured/Release | \ - egrep "binary-$ARCH/|source/" | \ - (while read m s file; do \ - dir="dists/$CODENAME-secured/`dirname $file`"; \ - test -d "$dir" || mkdir -p $dir; \ - cp $MIRROR/dists/$CODENAME/$file $dir/; \ - done) - # Create the files.list.gz - (find pool/ -type f; find dists/ -type f) | \ - gzip --best -c > dists/$CODENAME-secured/files.list.gz -fi - -if [ -n "$NONUS" -a -e "$NONUS/dists/$CODENAME/non-US/Release" ]; then - if [ ! -d "dists/$CODENAME-secured/non-US" ]; then - mkdir dists/$CODENAME-secured/non-US - fi - # Copy Release and Release.gpg files - cp $NONUS/dists/$CODENAME/non-US/Release dists/$CODENAME-secured/non-US/ - if [ -e "$NONUS/dists/$CODENAME/non-US/Release.gpg" ]; then - cp $NONUS/dists/$CODENAME/non-US/Release.gpg \ - dists/$CODENAME-secured/non-US/ - fi - # Include files listed in Release that corresponds to our arch - grep "^ " dists/$CODENAME-secured/non-US/Release | \ - egrep "binary-$ARCH/|source/" | \ - (while read m s file; do \ - dir="dists/$CODENAME-secured/non-US/`dirname $file`"; \ - test -d "$dir" || mkdir -p $dir; \ - cp $NONUS/dists/$CODENAME/non-US/$file $dir/; \ - done) - # The non-US files.list.gz is the same than the main one - ln -sf ../files.list.gz dists/$CODENAME-secured/non-US/files.list.gz -fi - - -fi # if secured diff --git a/tools/make_disc_trees.pl b/tools/make_disc_trees.pl index 1043653a..2a4c8566 100755 --- a/tools/make_disc_trees.pl +++ b/tools/make_disc_trees.pl @@ -53,46 +53,17 @@ my $MiB = 1048576; my $MB = 1000000; my $blocksize = 2048; my ($maxdiskblocks, $diskdesc); +my $cddir; my $disktype = $ENV{'DISKTYPE'}; - -# Calculate the maximum number of 2K blocks in the output images -if ($disktype eq "BC") { - $maxdiskblocks = int(680 * $MB / $blocksize); - $diskdesc = "businesscard"; -} elsif ($disktype eq "NETINST") { - $maxdiskblocks = int(680 * $MB / $blocksize); - $diskdesc = "netinst"; -} elsif ($disktype =~ /CD$/) { - $maxdiskblocks = int(680 * $MB / $blocksize); - $diskdesc = "650MiB CD"; -} elsif ($disktype eq "CD700") { - $maxdiskblocks = int(737 * $MB / $blocksize); - $diskdesc = "700MiB CD"; -} elsif ($disktype eq "DVD") { - $maxdiskblocks = int(4700 * $MB / $blocksize); - $diskdesc = "4.7GB CD"; -} elsif ($disktype eq "CUSTOM") { - $maxdiskblocks = $ENV{'CUSTOMSIZE'} || die "Need to specify a custom size for the CUSTOM disktype\n"; - $diskdesc = "User-supplied size"; -} - -$ENV{'MAXDISKBLOCKS'} = $maxdiskblocks; -$ENV{'DISKDESC'} = $diskdesc; - my $size_swap_check; my $hfs_extra = 0; my $hfs_mult = 1; -# How full should we let the disc get before we stop estimating and -# start running mkisofs? -# Cope with HFS-hybrid disks using extra space for the HFS metadata -$size_swap_check = $maxdiskblocks - (40 * $MB / $blocksize); - -# Add space for extra HFS crap +# Space calculation for extra HFS crap if ($archlist =~ /m68k/ || $archlist =~ /powerpc/) { - $hfs_extra = int($maxdiskblocks * 8 / $blocksize); - $hfs_mult = 1.1; + $hfs_extra = int($maxdiskblocks * 8 / $blocksize); + $hfs_mult = 1.1; } # And count how many packages added since the last size check was done @@ -112,6 +83,122 @@ if (defined ($ENV{'DEBOOTSTRAP_SCRIPT'})) { $debootstrap_script = $ENV{'DEBOOTSTRAP_SCRIPT'}; } +chdir $bdir; + +# Size calculation is slightly complicated: +# +# 1. At the start, ask mkisofs for a size so far (including all the +# stuff in the initial tree like docs and boot stuff +# +# 2. After that, add_packages will tell us the sizes of the files it +# has added. This will not include directories / metadata so is +# only a rough guess, but it's a _cheap_ guess +# +# 3. Once we get >90% of the max size we've been configured with, +# start asking mkisofs after each package addition. This will +# be slow, but we want to be exact at the end + +$cddir = "$bdir/CD$disknum"; +get_disc_size(); +print "Starting to lay out packages into $disktype ($diskdesc) images: $maxdiskblocks 2K-blocks maximum per image\n"; + +open(INLIST, "$bdir/packages") || die "No packages file!\n"; +while (defined (my $pkg = )) { + chomp $pkg; + $cddir = "$bdir/CD$disknum"; + my $opt; + if (! -d $cddir) { + if (($maxcds > 0 ) && ($disknum > $maxcds)) { + print LOG "Disk $disknum is beyond the configured MAXCDS of $maxcds; exiting now...\n"; + $max_done = 1; + last; + } + print LOG "Starting new disc $disknum at " . `date` . "\n"; + + start_disc(); + + print " Placing packages into image $disknum\n"; + if ( -e "$bdir/$disknum.mkisofs_opts" ) { + open(OPTS, "<$bdir/$disknum.mkisofs_opts"); + while (defined($opt = )) { + chomp $opt; + $mkisofs_opts = "$mkisofs_opts $opt"; + } + close(OPTS); + } else { + $mkisofs_opts = ""; + } + if ( -e "$bdir/$disknum.mkisofs_dirs" ) { + open(OPTS, "<$bdir/$disknum.mkisofs_dirs"); + while (defined($opt = )) { + chomp $opt; + $mkisofs_dirs = "$mkisofs_dirs $opt"; + } + close(OPTS); + } else { + $mkisofs_dirs = ""; + } + + $size_check = "$mkisofs_check $mkisofs_opts $mkisofs_dirs"; + $size=`$size_check $cddir`; + chomp $size; + $size += $hfs_extra; + print LOG "CD $disknum: size is $size before starting to add packages\n"; + if (defined($overflowpkg)) { + print LOG "Starting with the package that failed on the last disc: $overflowpkg\n"; + $guess_size = int($hfs_mult * add_packages($cddir, $overflowpkg)); + $size += $guess_size; + print LOG "CD $disknum: GUESS_TOTAL is $size after adding $overflowpkg\n"; + undef $overflowpkg; + $pkgs_this_cd = 1; + $pkgs_done++; + } + } + + $guess_size = int($hfs_mult * add_packages($cddir, $pkg)); + $size += $guess_size; + print LOG "CD $disknum: GUESS_TOTAL is $size after adding $pkg\n"; + if (($size > $maxdiskblocks) || + (($size > $size_swap_check) && + ($count_since_last_check > $size_check_period))) { + $count_since_last_check = 0; + $size = `$size_check $cddir`; + chomp $size; + print LOG "CD $disknum: Real current size is $size blocks after adding $pkg\n"; + } + if ($size > $maxdiskblocks) { + print LOG "CD $disknum over-full ($size > $maxdiskblocks). Rollback!\n"; + $guess_size = int($hfs_mult * add_packages("--rollback", $cddir, $pkg)); + $size=`$size_check $cddir`; + chomp $size; + print LOG "CD $disknum: Real current size is $size blocks after rolling back $pkg\n"; + + finish_disc($cddir, ""); + + # Put this package first on the next disc + $overflowpkg = $pkg; + + # And reset, to start the next disc + $size = 0; + $disknum++; + } else { + $pkgs_this_cd++; + $pkgs_done++; + $count_since_last_check++; + } +} +close(INLIST); + +if ($max_done == 0) { + finish_disc($cddir, " (not)"); +} + +print LOG "Finished: $pkgs_done packages placed\n"; +print "Finished: $pkgs_done packages placed\n"; +system("date >> $log"); + +close(LOG); + ############################################# # # Local helper functions @@ -203,6 +290,68 @@ sub md5_files_for_md5sum { } } +sub get_disc_size { + my $hook; + my $error = 0; + my $reserved = 0; + + if (defined($ENV{'RESERVED_BLOCKS_HOOK'})) { + $hook = $ENV{'RESERVED_BLOCKS_HOOK'}; + print " Calling reserved_blocks hook: $hook\n"; + $reserved = `$hook $tdir $mirror $disknum $cddir \"$archlist\"`; + chomp $reserved; + if ($reserved eq "") { + $reserved = 0; + } + print " Reserving $reserved blocks on CD $disknum\n"; + } + + # Calculate the maximum number of 2K blocks in the output images + if ($disktype eq "BC") { + $maxdiskblocks = int(680 * $MB / $blocksize) - $reserved; + $diskdesc = "businesscard"; + } elsif ($disktype eq "NETINST") { + $maxdiskblocks = int(680 * $MB / $blocksize) - $reserved; + $diskdesc = "netinst"; + } elsif ($disktype =~ /CD$/) { + $maxdiskblocks = int(680 * $MB / $blocksize) - $reserved; + $diskdesc = "650MiB CD"; + } elsif ($disktype eq "CD700") { + $maxdiskblocks = int(737 * $MB / $blocksize) - $reserved; + $diskdesc = "700MiB CD"; + } elsif ($disktype eq "DVD") { + $maxdiskblocks = int(4700 * $MB / $blocksize) - $reserved; + $diskdesc = "4.7GB CD"; + } elsif ($disktype eq "CUSTOM") { + $maxdiskblocks = $ENV{'CUSTOMSIZE'} - $reserved || + die "Need to specify a custom size for the CUSTOM disktype\n"; + $diskdesc = "User-supplied size"; + } + + $ENV{'MAXDISKBLOCKS'} = $maxdiskblocks; + $ENV{'DISKDESC'} = $diskdesc; + + # How full should we let the disc get before we stop estimating and + # start running mkisofs? + $size_swap_check = $maxdiskblocks - (40 * $MB / $blocksize); +} + +sub start_disc { + + system("start_new_disc $basedir $mirror $tdir $codename \"$archlist\" $disknum"); + + get_disc_size(); + + # Grab all the early stuff, apart from dirs that will change later + print " Starting the md5sum.txt file\n"; + chdir $cddir; + system("find . -type f | grep -v -e ^\./\.disk -e ^\./dists | xargs md5sum >> md5sum.txt"); + chdir $bdir; + + $mkisofs_opts = ""; + $mkisofs_dirs = ""; +} + sub finish_disc { my $cddir = shift; my $not = shift; @@ -210,6 +359,15 @@ sub finish_disc { my $ok = 0; my $bytes = 0; my $ctx; + my $hook; + my $error = 0; + + if (defined($ENV{'DISC_FINISH_HOOK'})) { + $hook = $ENV{'DISC_FINISH_HOOK'}; + print " Calling disc_finish hook: $hook\n"; + $error = system("$hook $tdir $mirror $disknum $cddir \"$archlist\""); + $error == 0 || die "DISC_FINISH_HOOK failed with error $error\n"; + } if (($disknum == 1) && !($archlist eq "source") && !($disktype eq "BC")) { foreach my $arch (@arches_nosrc) { @@ -253,6 +411,13 @@ sub finish_disc { system("mv -f md5sum.txt.tmp md5sum.txt"); chdir $bdir; + if (defined($ENV{'DISC_END_HOOK'})) { + $hook = $ENV{'DISC_END_HOOK'}; + print " Calling disc_end hook: $hook\n"; + $error = system("$hook $tdir $mirror $disknum $cddir \"$archlist\""); + $error == 0 || die "DISC_END_HOOK failed with error $error\n"; + } + $size = `$size_check $cddir`; chomp $size; $bytes = $size * $blocksize; @@ -260,127 +425,3 @@ sub finish_disc { print " CD $disknum$not filled with $pkgs_this_cd packages, $size blocks, $bytes bytes\n"; system("date >> $log"); } - -chdir $bdir; - -# Size calculation is slightly complicated: -# -# 1. At the start, ask mkisofs for a size so far (including all the -# stuff in the initial tree like docs and boot stuff -# -# 2. After that, add_packages will tell us the sizes of the files it -# has added. This will not include directories / metadata so is -# only a rough guess, but it's a _cheap_ guess -# -# 3. Once we get >90% of the max size we've been configured with, -# start asking mkisofs after each package addition. This will -# be slow, but we want to be exact at the end - -print "Starting to lay out packages into $disktype ($diskdesc) images: $maxdiskblocks 2K-blocks maximum per image\n"; - -my $cddir; - -open(INLIST, "$bdir/packages") || die "No packages file!\n"; -while (defined (my $pkg = )) { - chomp $pkg; - $cddir = "$bdir/CD$disknum"; - my $opt; - if (! -d $cddir) { - if (($maxcds > 0 ) && ($disknum > $maxcds)) { - print LOG "Disk $disknum is beyond the configured MAXCDS of $maxcds; exiting now...\n"; - $max_done = 1; - last; - } - print LOG "Starting new disc $disknum at " . `date` . "\n"; - system("start_new_disc $basedir $mirror $tdir $codename \"$archlist\" $disknum"); - - # Grab all the early stuff, apart from dirs that will change later - print " Starting the md5sum.txt file\n"; - chdir $cddir; - system("find . -type f | grep -v -e ^\./\.disk -e ^\./dists | xargs md5sum > md5sum.txt"); - chdir $bdir; - - $mkisofs_opts = ""; - $mkisofs_dirs = ""; - - print " Placing packages into image $disknum\n"; - if ( -e "$bdir/$disknum.mkisofs_opts" ) { - open(OPTS, "<$bdir/$disknum.mkisofs_opts"); - while (defined($opt = )) { - chomp $opt; - $mkisofs_opts = "$mkisofs_opts $opt"; - } - close(OPTS); - } else { - $mkisofs_opts = ""; - } - if ( -e "$bdir/$disknum.mkisofs_dirs" ) { - open(OPTS, "<$bdir/$disknum.mkisofs_dirs"); - while (defined($opt = )) { - chomp $opt; - $mkisofs_dirs = "$mkisofs_dirs $opt"; - } - close(OPTS); - } else { - $mkisofs_dirs = ""; - } - - $size_check = "$mkisofs_check $mkisofs_opts $mkisofs_dirs"; - $size=`$size_check $cddir`; - chomp $size; - $size += $hfs_extra; - print LOG "CD $disknum: size is $size before starting to add packages\n"; - if (defined($overflowpkg)) { - print LOG "Starting with the package that failed on the last disc: $overflowpkg\n"; - $guess_size = int($hfs_mult * add_packages($cddir, $overflowpkg)); - $size += $guess_size; - print LOG "CD $disknum: GUESS_TOTAL is $size after adding $overflowpkg\n"; - undef $overflowpkg; - $pkgs_this_cd = 1; - $pkgs_done++; - } - } - - $guess_size = int($hfs_mult * add_packages($cddir, $pkg)); - $size += $guess_size; - print LOG "CD $disknum: GUESS_TOTAL is $size after adding $pkg\n"; - if (($size > $size_swap_check) && ($count_since_last_check > $size_check_period)) { - $count_since_last_check = 0; - $size = `$size_check $cddir`; - chomp $size; - print LOG "CD $disknum: Real current size is $size blocks after adding $pkg\n"; - } - if ($size > $maxdiskblocks) { - print LOG "CD $disknum over-full ($size > $maxdiskblocks). Rollback!\n"; - $guess_size = int($hfs_mult * add_packages("--rollback", $cddir, $pkg)); - $size=`$size_check $cddir`; - chomp $size; - print LOG "CD $disknum: Real current size is $size blocks after rolling back $pkg\n"; - - finish_disc($cddir, ""); - - # Put this package first on the next disc - $overflowpkg = $pkg; - - # And reset, to start the next disc - $size = 0; - $disknum++; - } else { - $pkgs_this_cd++; - $pkgs_done++; - $count_since_last_check++; - } -} -close(INLIST); - -if ($max_done == 0) { - finish_disc($cddir, " (not)"); -} - -print LOG "Finished: $pkgs_done packages placed\n"; -print "Finished: $pkgs_done packages placed\n"; -system("date >> $log"); - -close(LOG); - - diff --git a/tools/start_new_disc b/tools/start_new_disc index c04aa2ce..34fa225a 100755 --- a/tools/start_new_disc +++ b/tools/start_new_disc @@ -81,6 +81,11 @@ if [ ! -d $CDDIR/.disk ] ; then mkdir $CDDIR/.disk fi +if [ "$DISC_START_HOOK"x != ""x ] ; then + echo " Calling start hook: $DISC_START_HOOK" + $DISC_START_HOOK $TDIR $MIRROR $DISKNUM $CDDIR "$ARCHES" +fi + echo " Generating the image label and volume id" echo -n $DISKINFO > $CDDIR/.disk/info echo -n "$DISKINFO ($DATE)" > $BDIR/$DISKNUM.diskinfo @@ -351,4 +356,9 @@ do fi done +if [ "$DISC_PKG_HOOK"x != ""x ] ; then + echo " Calling pkg hook: $DISC_PKG_HOOK" + $DISC_PKG_HOOK $TDIR $MIRROR $DISKNUM $CDDIR "$ARCHES" +fi + exit 0