Peppermint
/
debian-cd-clone
mirror of https://salsa.debian.org/images-team/debian-cd.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

* Update documentation to at least vaguely match how things work. 

* Added support for hook scripts in a few places around the CD
 	creation process, to help people customise their CDs. Quite a
	lot of refactoring needed in make_disc_trees.pl needed to do that.
  * Removed more cruft:
    + old defs in CONF.sh
    + old defs and rules in the Makefile
    + tools/add_secured
This commit is contained in:
Steve McIntyre 2006-12-19 01:58:41 +00:00
parent 7741fc3c83
commit b600a32f13
9 changed files with 401 additions and 483 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

85
CONF.sh
View File

@ -3,14 +3,11 @@
#
# Unset all optional variables first to start from a clean state
unset NONUS || true
unset FORCENONUSONCD1 || true
unset NONFREE || true
unset CONTRIB || true
unset EXTRANONFREE || true
unset LOCAL || true
unset LOCALDEBS || true
unset SECURED || true
unset SECURITY || true
unset BOOTDIR || true
unset SYMLINK || true
@ -29,11 +26,7 @@ unset JIGDOFALLBACKURLS || true
unset JIGDOINCLUDEURLS || true
unset JIGDOSCRIPT || true
unset JIGDO_OPTS || true
unset DEFBINSIZE || true
unset DEFSRCSIZE || true
unset FASTSUMS || true
unset PUBLISH_URL || true
unset PUBLISH_NONUS_URL || true
unset PUBLISH_PATH || true
unset UDEB_INCLUDE || true
unset UDEB_EXCLUDE || true
@ -99,14 +92,6 @@ fi
# Paths to the mirrors
export MIRROR=/mirror/debian
# Comment the following line if you don't have/want non-US
#export NONUS=/ftp/debian-non-US
# And this option will make you 2 copies of CD1 - one with all the
# non-US packages on it, one with none. Useful if you're likely to
# need both.
#export FORCENONUSONCD1=1
# Path of the temporary directory
export TDIR=/mirror/tmp
@ -137,11 +122,6 @@ export CONTRIB=1
# containing dists/$CODENAME/local/binary-$ARCH
# export LOCALDEBS=/home/joey/debian/va/debian
# If you want a <codename>-secured tree with a copy of the signed
# Release.gpg and files listed by this Release file, then
# uncomment this line
# export SECURED=1
# Where to find the security patches. This directory should be the
# top directory of a security.debian.org mirror.
#export SECURITY="$TOPDIR"/debian/debian-security
@ -166,7 +146,7 @@ export CONTRIB=1
export ISOLINUX=1
# uncomment this to if you want to see more of what the Makefile is doing
export VERBOSE_MAKE=1
#export VERBOSE_MAKE=1
# uncoment this to make build_all.sh try to build a simple CD image if
# the proper official CD run does not work
@ -180,7 +160,7 @@ ATTEMPT_FALLBACK=yes
# CD700: (semi-)standard 80-min CD (700 MiB)
# DVD: standard 4.7 GB DVD
# CUSTOM: up to you - specify a size to go with it (in 2K blocks)
export DISKTYPE=DVD
export DISKTYPE=CD
#export DISKTYPE=CUSTOM
#export CUSTOMSIZE=XXXX
@ -213,10 +193,6 @@ export NORECOMMENDS=1
# jigdo stuff is made; needs temporary space as big as the biggest image.
export DOJIGDO=1
# jigdo-file command & options
# Note: building the cache takes hours, so keep it around for the next run
#export JIGDOCMD="/usr/local/bin/jigdo-file --cache=$HOME/jigdo-cache.db"
#
# HTTP/FTP URL for directory where you intend to make the templates
# available. You should not need to change this; the default value ""
# means "template in same dir as the .jigdo file", which is usually
@ -251,15 +227,10 @@ export JIGDOINCLUDEURLS="http://cdimage.debian.org/debian-cd/debian-servers.jigd
# data.
#export JIGDOSCRIPT="myscript"
# If set, use the md5sums from the main archive, rather than calculating
# them locally
export FASTSUMS=1
# A couple of things used only by publish_cds, so it can tweak the
# jigdo files, and knows where to put the results.
# You need to run publish_cds manually, it is not run by the Makefile.
export PUBLISH_URL="http://cdimage.debian.org/jigdo-area"
export PUBLISH_NONUS_URL="http://non-US.cdimage.debian.org/jigdo-area"
export PUBLISH_PATH="/home/jigdo-area/"
# Specify files and directories to *exclude* from jigdo processing. These
@ -315,7 +286,7 @@ done
#export KERNEL_PARAMS="DEBCONF_PRIORITY=critical"
# If set, limits the number of binary CDs to produce.
# export MAXCDS=1
export MAXCDS=1
# If set, overrides the boot picture used.
#export SPLASHPNG="$BASEDIR/data/$CODENAME/splash-img.png"
@ -328,7 +299,7 @@ done
# If so we will link to them on the web site.
export OMIT_RELEASE_NOTES=1
# Set this to override the defaul location
# Set this to override the default location
#export RELEASE_NOTES_LOCATION="http://www.debian.org/releases/$CODENAME"
case "$OFFICIAL" in
@ -342,3 +313,51 @@ case "$OFFICIAL" in
export OFFICIAL_VAL=0
;;
esac
##################################
# LOCAL HOOK DEFINITIONS
##################################
#
# Set these to point to scripts/programs to be called at various
# points in the debian-cd image-making process. This is the ideal place
# to customise what's on the CDs, for example to add extra files or
# modify existing ones. Each will be called with the arguments in order:
#
# $TDIR (the temporary dir containing the build tree)
# $MIRROR (the location of the mirror)
# $DISKNUM (the image number in the set)
# $CDDIR (the root of the temp disc tree)
# $ARCHES (the set of architectures chosen)
#
# BE CAREFUL about what you do at each point: in the first couple of
# cases, files and directories you're looking to use may not exist yet,
# you may need to worry about adding entries into md5sum.txt yourself
# and (in the last couple of cases) if you add any extra files you may
# end up over-filling the disc. If you *do* need to add files at the end
# of the process, see RESERVED_BLOCKS_HOOK below. It's strongly
# recommended to do this kind of customisation up-front if you can, it's
# much simpler that way!
# The disc_start hook. This will be called near the beginning of the
# start_new_disc script, just after the directory tree has been created
# but before any files have been added
#export DISC_START_HOOK=/bin/true
# The disc_pkg hook. This will be called just after the
# start_new_disc script has finished, just before make_disc_trees.pl
# starts to add package files.
#export DISC_PKG_HOOK=/bin/true
# The reserved_blocks hook; if set, this script should print the
# number of 2K blocks that need to be reserved for data to be added
# *after* a disc tree is filled with packages.
#export RESERVED_BLOCKS_HOOK=/bin/true
# The disc_finish hook. This will be called once a disc image is full,
# just after the last package rollback but before the last bits of
# cleanup are done on the temp disc tree
#export DISC_FINISH_HOOK=/bin/true
# The disc_end hook. This will be called *right* at the end of the
# image-making process in make_disc_trees.pl.
#export DISC_END_HOOK=/bin/true

48
Makefile
View File

@ -51,14 +51,9 @@ endif
## Internal variables
apt=$(BASEDIR)/tools/apt-selection
list2cds=$(BASEDIR)/tools/list2cds
scansources=$(BASEDIR)/tools/scansources
add_files=$(BASEDIR)/tools/add_files
add_secured=$(BASEDIR)/tools/add_secured
md5sum=md5sum
fastsums=$(BASEDIR)/tools/fast_sums
jigdo_cleanup=$(BASEDIR)/tools/jigdo_cleanup
grab_md5=$(BASEDIR)/tools/grab_md5
dedicated-src=$(BASEDIR)/tools/dedicated_source
make_image=$(BASEDIR)/tools/make_image
merge_package_lists=$(BASEDIR)/tools/merge_package_lists
update_popcon=$(BASEDIR)/tools/update_popcon
@ -165,7 +160,7 @@ clean: ok dir-clean
dir-clean:
$(Q)rm -rf $(BDIR)/CD[1234567890]*
$(Q)rm -f $(BDIR)/*.filelist*
$(Q)rm -f $(BDIR)/packages-stamp $(BDIR)/upgrade-stamp $(BDIR)/secured-stamp $(BDIR)/md5-check
$(Q)rm -f $(BDIR)/packages-stamp $(BDIR)/upgrade-stamp $(BDIR)/md5-check
# Completely cleans the current arch tree
realclean: distclean
@ -370,47 +365,8 @@ need-complete-mirror:
exit 1; \
fi
dedicated-src: ok
# false
## EXTRAS ##
# Launch the extras scripts correctly for customizing the CDs
extras: bin-extras
bin-extras: ok
$(Q)if [ -z "$(DIR)" -o -z "$(CD)" -o -z "$(ROOTSRC)" ]; then \
echo "Give me more parameters (DIR, CD and ROOTSRC are required)."; \
false; \
fi
@echo "Adding dirs '$(DIR)' from '$(ROOTSRC)' to '$(BDIR)/CD$(CD)'" ...
$(Q)$(add_files) $(BDIR)/CD$(CD) $(ROOTSRC) $(DIR)
src-extras:
$(Q)if [ -z "$(DIR)" -o -z "$(CD)" -o -z "$(ROOTSRC)" ]; then \
echo "Give me more parameters (DIR, CD and ROOTSRC are required)."; \
false; \
fi
@echo "Adding dirs '$(DIR)' from '$(ROOTSRC)' to '$(BDIR)/CD$(CD)'" ...
$(Q)$(add_files) $(BDIR)/CD$(CD) $(ROOTSRC) $(DIR)
## IMAGE BUILDING ##
# Generate $CODENAME-secured tree with Packages and Release(.gpg) files
# from the official tree
# Complete the Release file from the normal tree
secured: bin-secured #src-secured
bin-secured: $(BDIR)/secured-stamp
$(BDIR)/secured-stamp:
@echo "Generating $(CODENAME)-secured on all the binary CDs ..."
$(Q)set -e; \
for file in $(BDIR)/*.packages; do \
dir=$${file%%.packages}; \
n=$${dir##$(BDIR)/}; \
dir=$(BDIR)/CD$$n; \
cd $$dir; \
$(add_secured); \
done
$(Q)touch $(BDIR)/secured-stamp
# DOJIGDO actions (for both binaries and source)
# 0 isofile
# 1 isofile + jigdo, cleanup_jigdo
@ -423,7 +379,7 @@ check-number-given:
@test -n "$(CD)" || (echo "Give me a CD=<num> parameter !" && false)
# Generate only one image number $(CD)
image: check-number-given image
image: check-number-given images
# Calculate the md5sums for the images (if available), or get from templates
imagesums:

171
README
View File

@ -1,16 +1,16 @@
YACS - Yet Another CD Script :-)
================================
(better known as debian-cd)
debian-cd
=========
Copyright 1999-2001 Raphaël Hertzog <hertzog@debian.org>
This set of tools is licensed under the General Public License
version 2 or any later version. You can find it in
Copyright 1999-2001 Raphaël Hertzog <hertzog@debian.org> and others,
2004-2006 Steve McIntyre <steve@einval.com>
This set of tools is licensed under the General Public License version
2 or any later version. You can find it in
/usr/share/common-licenses/GPL on a Debian GNU system.
Thanks to Steve McIntyre <stevem@chiark.greenend.org.uk> for his
work on slink_cd/debian_cd. Some ideas come from his script.
Some of the ideas here (a loooong time ago) came from Steve McIntyre's
slink_cd script.
Thanks to all the contributors on the debian-cd mailing list.
Thanks also to all the contributors on the debian-cd mailing list.
What is needed?
===============
@ -20,9 +20,10 @@ Software:
- perl (>= 5.004)
- bash (or another POSIX shell)
- make
- mkisofs/mkhybrid (mkisofs also provides the isoinfo binary used by the
Pseudo Image Kit)
- cpp
- mkisofs/genisoimage
- the perl Digest::MD5 module
- the perl Compress::Zlib module
- lynx (for text version of README.html) and todos from sysutils
to convert docs to DOS format (although you can rip that out, too)
- if you want to generate jigdo files: jigdo-file (see below)
@ -46,10 +47,7 @@ $ sensible-editor CONF.sh
$ . CONF.sh
$ make distclean
$ make status
$ make list COMPLETE=1 SIZELIMIT1=625000000 SRCSIZELIMIT=665000000
$ make official_images
[ alternatively, if you only want binary images:
$ make bin-official_images ]
However, you really should consider reading further for more information.
You can also take a look at build.sh and build_all.sh for an automatized
@ -103,33 +101,25 @@ The process of building a CD is composed of the following steps:
- now you can decide what you want on your CDs
$ make list TASK=tasks/debian-2.2 COMPLETE=1
$ make packagelists TASK=tasks/debian-2.2 COMPLETE=1
or
$ make list TASK=tasks/gnome COMPLETE=0 SIZELIMIT=576716800
$ make packagelists TASK=tasks/gnome COMPLETE=0
or
$ export NONFREE=1; make list TASK=tasks/your-task-here COMPLETE=1
$ export NONFREE=1; make packagelists TASK=tasks/your-task-here COMPLETE=1
or for something like an official image for the USA (without non-US &
non-free) :
$ make list COMPLETE=1 SIZELIMIT1=576716800
$ make packagelists COMPLETE=1
.... take a look at the file tasks/* to see the options you can have :)
You can change the behaviour of this command with the following
variables:
- if NONUS is set, then packages from non-US will be allowed (the value
of NONUS must be the path to the non-US mirror if you have one)
- if FORCENONUSONCD1 is set also, then packages will be
rearranged so that non-US packages are all on CD#1. This
includes the non-free ones if you specify NONFREE. Then 2 separate
copies of CD#1 will be produced, identical in every respect except
for the inclusion/lack of non-US packages. The same happens for
the source CDs when they are built.
- if NONFREE is set, then packages from non-free will be allowed
(NONFREE must be exported to all sub-shells)
- if EXTRANONFREE is set, then non-free packages will be included
@ -138,23 +128,30 @@ The process of building a CD is composed of the following steps:
(EXTRANONFREE must be exported to all sub-shells)
- if COMPLETE is set, all packages that are not listed in the
selected task file will be included at the end
- if SIZELIMIT is set, it will be used as the maximum size that
we can put into each CD
- if SIZELIMIT<X> (with <X> beeing a integer) is set, it will be used
as the maximum size of the X'th binary CD. SIZELIMIT<X> overrides
SIZELIMIT ...
- if SRCSIZELIMIT is set, it's used as the maximum size for source CD
- if SRCSIZELIMIT<X> (with <X> beeing an integer) is set, it will be
used as the maximum size of the X'th source CD
- setting INSTALLER_CD will use an appropriate task file for
building small CDs (businesscard and netinst)
This target calls the targets "bin-list" and "src-list" that can be used to
build only binary CDs or only source CDs.
- now, we'll start making temporary trees:
- it may be time to add the disks-arch stuff and to make the CDs bootable:
$ make image-trees
$ make bootable
This will first work sort the list of packages for each architecture
into order so that standard, required, important and base packages
are placed first, then other packages will be added in the order
given modulo dependency ordering. Once the sorted list is created,
the different architecture lists will be merged (if more than one
architecture is selected).
This affects only the binary CDs.
Then the code will start laying out temporary directory trees for
the CDs. In order, this includes the following steps:
- Creating an empty directory layout
- Generating an image label and volume ID and other metadata such as
debian-installer information files
- Add documentation and installation/upgrade tools
- Add Release files and other archive metadata
- Make the image bootable for the selected architecture(s)
- Start generating the md5sum.txt file
If you want to use boot-floppies built by yourself you can add
a parameter BOOTDISKS=<dir> which specifies the directory where
@ -163,57 +160,40 @@ The process of building a CD is composed of the following steps:
mirror. Your boot-floppies must also be on the same partition as
your mirror and temporary dir (hardlinks are used here too).
Size calculation is imprecise for bootable CDs (usually the first CDs
in the set), so you may need to manually adjust SIZELIMIT* variables
to account for the size of the booting stuff used.
Once the disc tree has all of this start data, we start filling the
directory trees with packages from the sorted list. The size of the
image to be created is set using DISKTYPE in CONF.sh; if the
standard sizes do not match what you're after, use DISKTYPE=CUSTOM
and specify your own size using CUSTOMSIZE. The algorithm is simple
for adding packages:
- now, we'll add the binary packages to the temporary tree:
- link the package into the temporary disc tree
- append the metadata to the appropriate Packages or Sources file
- add md5sum information for the added file(s) to the md5sum.txt file
$ make packages
This continues until the temporary tree grows one package *too
large* for the selected image size. At that point, we roll back the
last set of changes associated with that package. Then:
- and we'll add the sources to the temporary tree:
- check if the disc contains all the packages needed to install a base system
- finish off the Release file, using the checksums of the
Packages/Sources files we generated
- finish off the md5sum.txt file
$ make sources
- if you want to install additional files:
$ make bin-extras CD=1 ROOTSRC=/home/ftp/ DIR=goodies/wordperfect
$ make src-extras CD=3 ROOTSRC=/home/ftp/ DIR=goodies/kernel-2.3
The first will copy /home/ftp/goodies/wordperfect/ to the first binary
CD. It will be in <root of the cd>/goodies/wordperfect/. You can call
make extras multiple times if you need more.
Please note that the files to be copied should be on the same partition
than your mirror (unless you use a symlink farm).
If you want to do customize your CD even more, you can use the hook
system. Read more about that below.
- We can add an md5sum.txt file on each CD to enable users to check their
files:
$ make md5list
This calls the targets 'bin-md5list' and 'src-md5list'. You can
choose to call only bin-md5list if you're building only binary images.
Next, we continue to the next disc tree, using the same process:
start it, copy packages in until they overflow, roll back and
finish. And repeat. Each time a package is found to be too large to
fit inside an image, it will be kept back and will (obviously) be
the first package placed into the next disc tree.
- now we can create the images:
$ make images
If you don't have enough space for all images, you can generate
only one image (of the second binary CD for example) with:
only one image (of the second CD for example) with:
$ make bin-image CD=2
Of course if you want to build all binary images you'll use:
$ make bin-images
The following will generate the source images:
$ make src-images
$ make image CD=2
Note: here we use "make images", but you could as well use
"make official_images" since the latter is the same as the former
@ -225,7 +205,6 @@ The process of building a CD is composed of the following steps:
$ make imagesums
Official images
===============
@ -281,16 +260,9 @@ for jigdo, merely a modification of the "images" targets. You can choose
only .iso generation (default), only .jigdo generation (for highly reduced
disk usage), or both .iso and .jigdo generation.
To generate the jigdo files and templates, you need the "jigdo-file" program,
which is available via the same URL.
Jigdo-file creates and maintains a cache/database file with checksums of
all files on your Debian mirror. The first time, this may take hours to be
generated (use "top" to see what's going on), so it's wise to keep the
cache in your homedir and not delete it ever.
Note that jigdo-file can easily use 60+ MB of working memory, so don't use
this on machines with less than 128 MB RAM.
To generate the jigdo files and templates, you need an
appropriately-patched version of mkisofs/genisoimage, as shipped in
Debian Etch.
The MD5SUMS file generated by the "imagesums" target will contain the MD5
checksums of all generated images, regardless of the DOJIGDO setting. If
@ -322,24 +294,15 @@ debian-cd allows you to automatically create a directory on disc which
is suitable for use as a fallback mirror. It is populated with hard
links to the archive contents. In CONF.sh, simply supply as
JIGDOFALLBACKPATH the name of the directory, and as JIGDOFALLBACKURLS
the URLs under which it will be made available (two separate URLs are
necessary, one for "Debian" and one for "Non-US").
the URLs under which it will be made available.
About the hook system
=====================
A hook script can be executed at different times. You can specify the
script by setting the HOOK variable to the script filename. It will
get 2 arguments, the first is the CD number. The second depends on
where/when the hook script is called. It can be 'before-scanpackages'
or 'before-mkisofs' (their values are explicit ...). When the script
is called, the current directory will be the temporary directory used for
the build (aka $TDIR/$CODENAME-$ARCH).
There are hooks only for binary CDs at the present time. If HOOK is not
set, it will look for a script $BASEDIR/tools/$CODENAME.hook.
A hook script can be executed at different times during the CD build
process to customise your CDs. You can specify the script by setting
the various HOOK variables in CONF.sh; look there for more information
about what hook points are available.
About the symlink farm
======================

26
README.devel
View File

@ -1,5 +1,5 @@
README for those who want to hack on yacs/debian-cd
---------------------------------------------------
README for those who want to hack on debian-cd
----------------------------------------------
Organisation :
--------------
@ -12,10 +12,9 @@ auth: svn+ssh://svn.debian.org/svn/debian-cd/trunk
anon: svn://svn.debian.org/debian-cd/trunk
web: http://svn.debian.org/wsvn/debian-cd
If you want to discuss anything related to the debian-cd development, you
can contact me directly or, better, you can mail the
debian-cd@lists.debian.org mailing list where all people interested in the
debian-cd development are subscribed.
If you want to discuss anything related to the debian-cd development,
mail the debian-cd@lists.debian.org mailing list where all people
interested in the debian-cd development are subscribed.
Technical details :
-------------------
@ -36,8 +35,8 @@ related) like the master file from boot-floppies and so on.
Each tool is self-documented, if you want to know what it
does read the sources (they are scripts).
Debugging YACS :
----------------
Debugging debian-cd :
---------------------
If you want to read more about what YACS is doing you can set
the VERBOSE environment variable to 1, 2 or 3 depending on the
@ -47,16 +46,14 @@ Some scripts generates their own log files in the temp dir. You
can take a look at them if you want to check for warnings
and so on.
Log files :
- $TDIR/$CODENAME-$ARCH/log.list2cds
- $TDIR/$CODENAME-SRC/log.cds2src
- $TDIR/$CODENAME/log.list2cds
- $TDIR/$CODENAME/make_disc_trees.log
TODO list :
-----------
+ put the doc directory only on the first binary CD
+ a way to force the inclusion of packages (even broken)
+ possibility to add project/* to the last CD
+ check the source files on the mirror (with Sources.gz)
when doing make mirrorcheck
Bugs / Problems :
-----------------
@ -75,7 +72,4 @@ The el-torito multiboot just displays a menu with each item
being the same "2.88Mb boot image". Both multiboot mechanism may
not work on older hardware with very old BIOSes however the el-torito
one works a bit better because it's usually able to boot the first choice
without displaying the menu then. This is a non-issue since people
may still boot on CD 2,3,4,5 with the usual boot method if the multiboot
doesn't work.
without displaying the menu then.

85
debian/CONF.sh vendored
View File

@ -3,14 +3,11 @@
#
# Unset all optional variables first to start from a clean state
unset NONUS || true
unset FORCENONUSONCD1 || true
unset NONFREE || true
unset CONTRIB || true
unset EXTRANONFREE || true
unset LOCAL || true
unset LOCALDEBS || true
unset SECURED || true
unset SECURITY || true
unset BOOTDIR || true
unset SYMLINK || true
@ -29,11 +26,7 @@ unset JIGDOFALLBACKURLS || true
unset JIGDOINCLUDEURLS || true
unset JIGDOSCRIPT || true
unset JIGDO_OPTS || true
unset DEFBINSIZE || true
unset DEFSRCSIZE || true
unset FASTSUMS || true
unset PUBLISH_URL || true
unset PUBLISH_NONUS_URL || true
unset PUBLISH_PATH || true
unset UDEB_INCLUDE || true
unset UDEB_EXCLUDE || true
@ -99,14 +92,6 @@ fi
# Paths to the mirrors
export MIRROR=/mirror/debian
# Comment the following line if you don't have/want non-US
#export NONUS=/ftp/debian-non-US
# And this option will make you 2 copies of CD1 - one with all the
# non-US packages on it, one with none. Useful if you're likely to
# need both.
#export FORCENONUSONCD1=1
# Path of the temporary directory
export TDIR=/mirror/tmp
@ -137,11 +122,6 @@ export CONTRIB=1
# containing dists/$CODENAME/local/binary-$ARCH
# export LOCALDEBS=/home/joey/debian/va/debian
# If you want a <codename>-secured tree with a copy of the signed
# Release.gpg and files listed by this Release file, then
# uncomment this line
# export SECURED=1
# Where to find the security patches. This directory should be the
# top directory of a security.debian.org mirror.
#export SECURITY="$TOPDIR"/debian/debian-security
@ -166,7 +146,7 @@ export CONTRIB=1
export ISOLINUX=1
# uncomment this to if you want to see more of what the Makefile is doing
export VERBOSE_MAKE=1
#export VERBOSE_MAKE=1
# uncoment this to make build_all.sh try to build a simple CD image if
# the proper official CD run does not work
@ -180,7 +160,7 @@ ATTEMPT_FALLBACK=yes
# CD700: (semi-)standard 80-min CD (700 MiB)
# DVD: standard 4.7 GB DVD
# CUSTOM: up to you - specify a size to go with it (in 2K blocks)
export DISKTYPE=DVD
export DISKTYPE=CD
#export DISKTYPE=CUSTOM
#export CUSTOMSIZE=XXXX
@ -213,10 +193,6 @@ export NORECOMMENDS=1
# jigdo stuff is made; needs temporary space as big as the biggest image.
export DOJIGDO=1
# jigdo-file command & options
# Note: building the cache takes hours, so keep it around for the next run
#export JIGDOCMD="/usr/local/bin/jigdo-file --cache=$HOME/jigdo-cache.db"
#
# HTTP/FTP URL for directory where you intend to make the templates
# available. You should not need to change this; the default value ""
# means "template in same dir as the .jigdo file", which is usually
@ -251,15 +227,10 @@ export JIGDOINCLUDEURLS="http://cdimage.debian.org/debian-cd/debian-servers.jigd
# data.
#export JIGDOSCRIPT="myscript"
# If set, use the md5sums from the main archive, rather than calculating
# them locally
export FASTSUMS=1
# A couple of things used only by publish_cds, so it can tweak the
# jigdo files, and knows where to put the results.
# You need to run publish_cds manually, it is not run by the Makefile.
export PUBLISH_URL="http://cdimage.debian.org/jigdo-area"
export PUBLISH_NONUS_URL="http://non-US.cdimage.debian.org/jigdo-area"
export PUBLISH_PATH="/home/jigdo-area/"
# Specify files and directories to *exclude* from jigdo processing. These
@ -315,7 +286,7 @@ done
#export KERNEL_PARAMS="DEBCONF_PRIORITY=critical"
# If set, limits the number of binary CDs to produce.
# export MAXCDS=1
export MAXCDS=1
# If set, overrides the boot picture used.
#export SPLASHPNG="$BASEDIR/data/$CODENAME/splash-img.png"
@ -328,7 +299,7 @@ done
# If so we will link to them on the web site.
export OMIT_RELEASE_NOTES=1
# Set this to override the defaul location
# Set this to override the default location
#export RELEASE_NOTES_LOCATION="http://www.debian.org/releases/$CODENAME"
case "$OFFICIAL" in
@ -342,3 +313,51 @@ case "$OFFICIAL" in
export OFFICIAL_VAL=0
;;
esac
##################################
# LOCAL HOOK DEFINITIONS
##################################
#
# Set these to point to scripts/programs to be called at various
# points in the debian-cd image-making process. This is the ideal place
# to customise what's on the CDs, for example to add extra files or
# modify existing ones. Each will be called with the arguments in order:
#
# $TDIR (the temporary dir containing the build tree)
# $MIRROR (the location of the mirror)
# $DISKNUM (the image number in the set)
# $CDDIR (the root of the temp disc tree)
# $ARCHES (the set of architectures chosen)
#
# BE CAREFUL about what you do at each point: in the first couple of
# cases, files and directories you're looking to use may not exist yet,
# you may need to worry about adding entries into md5sum.txt yourself
# and (in the last couple of cases) if you add any extra files you may
# end up over-filling the disc. If you *do* need to add files at the end
# of the process, see RESERVED_BLOCKS_HOOK below. It's strongly
# recommended to do this kind of customisation up-front if you can, it's
# much simpler that way!
# The disc_start hook. This will be called near the beginning of the
# start_new_disc script, just after the directory tree has been created
# but before any files have been added
#export DISC_START_HOOK=/bin/true
# The disc_pkg hook. This will be called just after the
# start_new_disc script has finished, just before make_disc_trees.pl
# starts to add package files.
#export DISC_PKG_HOOK=/bin/true
# The reserved_blocks hook; if set, this script should print the
# number of 2K blocks that need to be reserved for data to be added
# *after* a disc tree is filled with packages.
#export RESERVED_BLOCKS_HOOK=/bin/true
# The disc_finish hook. This will be called once a disc image is full,
# just after the last package rollback but before the last bits of
# cleanup are done on the temp disc tree
#export DISC_FINISH_HOOK=/bin/true
# The disc_end hook. This will be called *right* at the end of the
# image-making process in make_disc_trees.pl.
#export DISC_END_HOOK=/bin/true

10
debian/changelog vendored
View File

@ -172,6 +172,16 @@ debian-cd (3.0.0) UNRELEASED; urgency=low
* CONF.sh: don't determine ARCHES if already set; don't export CPU and
KERNEL.
[ Steve McIntyre ]
* Update documentation to at least vaguely match how things work.
* Added support for hook scripts in a few places around the CD
creation process, to help people customise their CDs. Quite a
lot of refactoring needed in make_disc_trees.pl needed to do that.
* Removed more cruft:
+ old defs in CONF.sh
+ old defs and rules in the Makefile
+ tools/add_secured
-- Frans Pop <fjp@debian.org> Mon, 18 Dec 2006 16:20:22 +0100
debian-cd (2.2.25) unstable; urgency=low

94
tools/add_secured
View File

@ -1,94 +0,0 @@
#!/bin/sh
# We are in the temp directory of a CD image
# aka $[SB]DIR/CD<x>/
# Everything is already in place (just md5sum.txt missing)
# Complete the "Release" file if it exists
if [ -e "dists/$CODENAME/Release" ]; then
# Add the MD5Sum field again
echo "MD5Sum:" >> dists/$CODENAME/Release
find dists/$CODENAME/ -name 'Packages' -o -name 'Packages.gz' \
-o -name 'Sources' -o -name 'Sources.gz' -o -name 'Release' | \
grep -v non-US/ | grep -v dists/$CODENAME/Release | \
sed -e "s#^dists/$CODENAME/##" | \
(while read file; do \
rfile="dists/$CODENAME/$file"; \
c=`wc -c < $rfile`; \
m=`md5sum < $rfile | cut -d" " -f1`; \
printf " %s %8d %s\n" $m $c $file; \
done) >> dists/$CODENAME/Release
# TODO: Should add SHA1 field here
# Check with libdigest-sha1-perl
fi
if [ -n "$NONUS" -a -e "dists/$CODENAME/non-US/Release" ]; then
# Add the MD5Sum field again
echo "MD5Sum:" >> dists/$CODENAME/non-US/Release
find dists/$CODENAME/non-US/ -name 'Packages' -o -name 'Packages.gz' \
-o -name 'Sources' -o -name 'Sources.gz' -o -name 'Release' | \
grep -v dists/$CODENAME/non-US/Release | \
sed -e "s#^dists/$CODENAME/non-US/##" | \
(while read file; do \
rfile="dists/$CODENAME/non-US/$file"; \
c=`wc -c < $rfile`; \
m=`md5sum < $rfile | cut -d" " -f1`; \
printf " %s %8d %s\n" $m $c $file; \
done) >> dists/$CODENAME/non-US/Release
# TODO: Should add SHA1 field here
# Check with libdigest-sha1-perl
fi
# Create the $CODENAME-secured tree by copying files from the mirror
# And including the Release file and its signature
if [ -n "$SECURED" ]; then
test -d "dists/$CODENAME-secured" || mkdir dists/$CODENAME-secured
if [ -e "$MIRROR/dists/$CODENAME/Release" ]; then
# Copy Release and Release.gpg files
cp $MIRROR/dists/$CODENAME/Release dists/$CODENAME-secured/
if [ -e "$MIRROR/dists/$CODENAME/Release.gpg" ]; then
cp $MIRROR/dists/$CODENAME/Release.gpg dists/$CODENAME-secured/
fi
# Create the .aptignr file in the standard tree
:> dists/$CODENAME/.aptignr
# Include files listed in Release that corresponds to our arch
grep "^ " dists/$CODENAME-secured/Release | \
egrep "binary-$ARCH/|source/" | \
(while read m s file; do \
dir="dists/$CODENAME-secured/`dirname $file`"; \
test -d "$dir" || mkdir -p $dir; \
cp $MIRROR/dists/$CODENAME/$file $dir/; \
done)
# Create the files.list.gz
(find pool/ -type f; find dists/ -type f) | \
gzip --best -c > dists/$CODENAME-secured/files.list.gz
fi
if [ -n "$NONUS" -a -e "$NONUS/dists/$CODENAME/non-US/Release" ]; then
if [ ! -d "dists/$CODENAME-secured/non-US" ]; then
mkdir dists/$CODENAME-secured/non-US
fi
# Copy Release and Release.gpg files
cp $NONUS/dists/$CODENAME/non-US/Release dists/$CODENAME-secured/non-US/
if [ -e "$NONUS/dists/$CODENAME/non-US/Release.gpg" ]; then
cp $NONUS/dists/$CODENAME/non-US/Release.gpg \
dists/$CODENAME-secured/non-US/
fi
# Include files listed in Release that corresponds to our arch
grep "^ " dists/$CODENAME-secured/non-US/Release | \
egrep "binary-$ARCH/|source/" | \
(while read m s file; do \
dir="dists/$CODENAME-secured/non-US/`dirname $file`"; \
test -d "$dir" || mkdir -p $dir; \
cp $NONUS/dists/$CODENAME/non-US/$file $dir/; \
done)
# The non-US files.list.gz is the same than the main one
ln -sf ../files.list.gz dists/$CODENAME-secured/non-US/files.list.gz
fi
fi # if secured

355
tools/make_disc_trees.pl
View File

@ -53,46 +53,17 @@ my $MiB = 1048576;
my $MB = 1000000;
my $blocksize = 2048;
my ($maxdiskblocks, $diskdesc);
my $cddir;
my $disktype = $ENV{'DISKTYPE'};
# Calculate the maximum number of 2K blocks in the output images
if ($disktype eq "BC") {
$maxdiskblocks = int(680 * $MB / $blocksize);
$diskdesc = "businesscard";
} elsif ($disktype eq "NETINST") {
$maxdiskblocks = int(680 * $MB / $blocksize);
$diskdesc = "netinst";
} elsif ($disktype =~ /CD$/) {
$maxdiskblocks = int(680 * $MB / $blocksize);
$diskdesc = "650MiB CD";
} elsif ($disktype eq "CD700") {
$maxdiskblocks = int(737 * $MB / $blocksize);
$diskdesc = "700MiB CD";
} elsif ($disktype eq "DVD") {
$maxdiskblocks = int(4700 * $MB / $blocksize);
$diskdesc = "4.7GB CD";
} elsif ($disktype eq "CUSTOM") {
$maxdiskblocks = $ENV{'CUSTOMSIZE'} || die "Need to specify a custom size for the CUSTOM disktype\n";
$diskdesc = "User-supplied size";
}
$ENV{'MAXDISKBLOCKS'} = $maxdiskblocks;
$ENV{'DISKDESC'} = $diskdesc;
my $size_swap_check;
my $hfs_extra = 0;
my $hfs_mult = 1;
# How full should we let the disc get before we stop estimating and
# start running mkisofs?
# Cope with HFS-hybrid disks using extra space for the HFS metadata
$size_swap_check = $maxdiskblocks - (40 * $MB / $blocksize);
# Add space for extra HFS crap
# Space calculation for extra HFS crap
if ($archlist =~ /m68k/ || $archlist =~ /powerpc/) {
$hfs_extra = int($maxdiskblocks * 8 / $blocksize);
$hfs_mult = 1.1;
$hfs_extra = int($maxdiskblocks * 8 / $blocksize);
$hfs_mult = 1.1;
}
# And count how many packages added since the last size check was done
@ -112,6 +83,122 @@ if (defined ($ENV{'DEBOOTSTRAP_SCRIPT'})) {
$debootstrap_script = $ENV{'DEBOOTSTRAP_SCRIPT'};
}
chdir $bdir;
# Size calculation is slightly complicated:
#
# 1. At the start, ask mkisofs for a size so far (including all the
# stuff in the initial tree like docs and boot stuff
#
# 2. After that, add_packages will tell us the sizes of the files it
# has added. This will not include directories / metadata so is
# only a rough guess, but it's a _cheap_ guess
#
# 3. Once we get >90% of the max size we've been configured with,
# start asking mkisofs after each package addition. This will
# be slow, but we want to be exact at the end
$cddir = "$bdir/CD$disknum";
get_disc_size();
print "Starting to lay out packages into $disktype ($diskdesc) images: $maxdiskblocks 2K-blocks maximum per image\n";
open(INLIST, "$bdir/packages") || die "No packages file!\n";
while (defined (my $pkg = <INLIST>)) {
chomp $pkg;
$cddir = "$bdir/CD$disknum";
my $opt;
if (! -d $cddir) {
if (($maxcds > 0 ) && ($disknum > $maxcds)) {
print LOG "Disk $disknum is beyond the configured MAXCDS of $maxcds; exiting now...\n";
$max_done = 1;
last;
}
print LOG "Starting new disc $disknum at " . `date` . "\n";
start_disc();
print " Placing packages into image $disknum\n";
if ( -e "$bdir/$disknum.mkisofs_opts" ) {
open(OPTS, "<$bdir/$disknum.mkisofs_opts");
while (defined($opt = <OPTS>)) {
chomp $opt;
$mkisofs_opts = "$mkisofs_opts $opt";
}
close(OPTS);
} else {
$mkisofs_opts = "";
}
if ( -e "$bdir/$disknum.mkisofs_dirs" ) {
open(OPTS, "<$bdir/$disknum.mkisofs_dirs");
while (defined($opt = <OPTS>)) {
chomp $opt;
$mkisofs_dirs = "$mkisofs_dirs $opt";
}
close(OPTS);
} else {
$mkisofs_dirs = "";
}
$size_check = "$mkisofs_check $mkisofs_opts $mkisofs_dirs";
$size=`$size_check $cddir`;
chomp $size;
$size += $hfs_extra;
print LOG "CD $disknum: size is $size before starting to add packages\n";
if (defined($overflowpkg)) {
print LOG "Starting with the package that failed on the last disc: $overflowpkg\n";
$guess_size = int($hfs_mult * add_packages($cddir, $overflowpkg));
$size += $guess_size;
print LOG "CD $disknum: GUESS_TOTAL is $size after adding $overflowpkg\n";
undef $overflowpkg;
$pkgs_this_cd = 1;
$pkgs_done++;
}
}
$guess_size = int($hfs_mult * add_packages($cddir, $pkg));
$size += $guess_size;
print LOG "CD $disknum: GUESS_TOTAL is $size after adding $pkg\n";
if (($size > $maxdiskblocks) ||
(($size > $size_swap_check) &&
($count_since_last_check > $size_check_period))) {
$count_since_last_check = 0;
$size = `$size_check $cddir`;
chomp $size;
print LOG "CD $disknum: Real current size is $size blocks after adding $pkg\n";
}
if ($size > $maxdiskblocks) {
print LOG "CD $disknum over-full ($size > $maxdiskblocks). Rollback!\n";
$guess_size = int($hfs_mult * add_packages("--rollback", $cddir, $pkg));
$size=`$size_check $cddir`;
chomp $size;
print LOG "CD $disknum: Real current size is $size blocks after rolling back $pkg\n";
finish_disc($cddir, "");
# Put this package first on the next disc
$overflowpkg = $pkg;
# And reset, to start the next disc
$size = 0;
$disknum++;
} else {
$pkgs_this_cd++;
$pkgs_done++;
$count_since_last_check++;
}
}
close(INLIST);
if ($max_done == 0) {
finish_disc($cddir, " (not)");
}
print LOG "Finished: $pkgs_done packages placed\n";
print "Finished: $pkgs_done packages placed\n";
system("date >> $log");
close(LOG);
#############################################
#
# Local helper functions
@ -203,6 +290,68 @@ sub md5_files_for_md5sum {
}
}
sub get_disc_size {
my $hook;
my $error = 0;
my $reserved = 0;
if (defined($ENV{'RESERVED_BLOCKS_HOOK'})) {
$hook = $ENV{'RESERVED_BLOCKS_HOOK'};
print " Calling reserved_blocks hook: $hook\n";
$reserved = `$hook $tdir $mirror $disknum $cddir \"$archlist\"`;
chomp $reserved;
if ($reserved eq "") {
$reserved = 0;
}
print " Reserving $reserved blocks on CD $disknum\n";
}
# Calculate the maximum number of 2K blocks in the output images
if ($disktype eq "BC") {
$maxdiskblocks = int(680 * $MB / $blocksize) - $reserved;
$diskdesc = "businesscard";
} elsif ($disktype eq "NETINST") {
$maxdiskblocks = int(680 * $MB / $blocksize) - $reserved;
$diskdesc = "netinst";
} elsif ($disktype =~ /CD$/) {
$maxdiskblocks = int(680 * $MB / $blocksize) - $reserved;
$diskdesc = "650MiB CD";
} elsif ($disktype eq "CD700") {
$maxdiskblocks = int(737 * $MB / $blocksize) - $reserved;
$diskdesc = "700MiB CD";
} elsif ($disktype eq "DVD") {
$maxdiskblocks = int(4700 * $MB / $blocksize) - $reserved;
$diskdesc = "4.7GB CD";
} elsif ($disktype eq "CUSTOM") {
$maxdiskblocks = $ENV{'CUSTOMSIZE'} - $reserved ||
die "Need to specify a custom size for the CUSTOM disktype\n";
$diskdesc = "User-supplied size";
}
$ENV{'MAXDISKBLOCKS'} = $maxdiskblocks;
$ENV{'DISKDESC'} = $diskdesc;
# How full should we let the disc get before we stop estimating and
# start running mkisofs?
$size_swap_check = $maxdiskblocks - (40 * $MB / $blocksize);
}
sub start_disc {
system("start_new_disc $basedir $mirror $tdir $codename \"$archlist\" $disknum");
get_disc_size();
# Grab all the early stuff, apart from dirs that will change later
print " Starting the md5sum.txt file\n";
chdir $cddir;
system("find . -type f | grep -v -e ^\./\.disk -e ^\./dists | xargs md5sum >> md5sum.txt");
chdir $bdir;
$mkisofs_opts = "";
$mkisofs_dirs = "";
}
sub finish_disc {
my $cddir = shift;
my $not = shift;
@ -210,6 +359,15 @@ sub finish_disc {
my $ok = 0;
my $bytes = 0;
my $ctx;
my $hook;
my $error = 0;
if (defined($ENV{'DISC_FINISH_HOOK'})) {
$hook = $ENV{'DISC_FINISH_HOOK'};
print " Calling disc_finish hook: $hook\n";
$error = system("$hook $tdir $mirror $disknum $cddir \"$archlist\"");
$error == 0 || die "DISC_FINISH_HOOK failed with error $error\n";
}
if (($disknum == 1) && !($archlist eq "source") && !($disktype eq "BC")) {
foreach my $arch (@arches_nosrc) {
@ -253,6 +411,13 @@ sub finish_disc {
system("mv -f md5sum.txt.tmp md5sum.txt");
chdir $bdir;
if (defined($ENV{'DISC_END_HOOK'})) {
$hook = $ENV{'DISC_END_HOOK'};
print " Calling disc_end hook: $hook\n";
$error = system("$hook $tdir $mirror $disknum $cddir \"$archlist\"");
$error == 0 || die "DISC_END_HOOK failed with error $error\n";
}
$size = `$size_check $cddir`;
chomp $size;
$bytes = $size * $blocksize;
@ -260,127 +425,3 @@ sub finish_disc {
print " CD $disknum$not filled with $pkgs_this_cd packages, $size blocks, $bytes bytes\n";
system("date >> $log");
}
chdir $bdir;
# Size calculation is slightly complicated:
#
# 1. At the start, ask mkisofs for a size so far (including all the
# stuff in the initial tree like docs and boot stuff
#
# 2. After that, add_packages will tell us the sizes of the files it
# has added. This will not include directories / metadata so is
# only a rough guess, but it's a _cheap_ guess
#
# 3. Once we get >90% of the max size we've been configured with,
# start asking mkisofs after each package addition. This will
# be slow, but we want to be exact at the end
print "Starting to lay out packages into $disktype ($diskdesc) images: $maxdiskblocks 2K-blocks maximum per image\n";
my $cddir;
open(INLIST, "$bdir/packages") || die "No packages file!\n";
while (defined (my $pkg = <INLIST>)) {
chomp $pkg;
$cddir = "$bdir/CD$disknum";
my $opt;
if (! -d $cddir) {
if (($maxcds > 0 ) && ($disknum > $maxcds)) {
print LOG "Disk $disknum is beyond the configured MAXCDS of $maxcds; exiting now...\n";
$max_done = 1;
last;
}
print LOG "Starting new disc $disknum at " . `date` . "\n";
system("start_new_disc $basedir $mirror $tdir $codename \"$archlist\" $disknum");
# Grab all the early stuff, apart from dirs that will change later
print " Starting the md5sum.txt file\n";
chdir $cddir;
system("find . -type f | grep -v -e ^\./\.disk -e ^\./dists | xargs md5sum > md5sum.txt");
chdir $bdir;
$mkisofs_opts = "";
$mkisofs_dirs = "";
print " Placing packages into image $disknum\n";
if ( -e "$bdir/$disknum.mkisofs_opts" ) {
open(OPTS, "<$bdir/$disknum.mkisofs_opts");
while (defined($opt = <OPTS>)) {
chomp $opt;
$mkisofs_opts = "$mkisofs_opts $opt";
}
close(OPTS);
} else {
$mkisofs_opts = "";
}
if ( -e "$bdir/$disknum.mkisofs_dirs" ) {
open(OPTS, "<$bdir/$disknum.mkisofs_dirs");
while (defined($opt = <OPTS>)) {
chomp $opt;
$mkisofs_dirs = "$mkisofs_dirs $opt";
}
close(OPTS);
} else {
$mkisofs_dirs = "";
}
$size_check = "$mkisofs_check $mkisofs_opts $mkisofs_dirs";
$size=`$size_check $cddir`;
chomp $size;
$size += $hfs_extra;
print LOG "CD $disknum: size is $size before starting to add packages\n";
if (defined($overflowpkg)) {
print LOG "Starting with the package that failed on the last disc: $overflowpkg\n";
$guess_size = int($hfs_mult * add_packages($cddir, $overflowpkg));
$size += $guess_size;
print LOG "CD $disknum: GUESS_TOTAL is $size after adding $overflowpkg\n";
undef $overflowpkg;
$pkgs_this_cd = 1;
$pkgs_done++;
}
}
$guess_size = int($hfs_mult * add_packages($cddir, $pkg));
$size += $guess_size;
print LOG "CD $disknum: GUESS_TOTAL is $size after adding $pkg\n";
if (($size > $size_swap_check) && ($count_since_last_check > $size_check_period)) {
$count_since_last_check = 0;
$size = `$size_check $cddir`;
chomp $size;
print LOG "CD $disknum: Real current size is $size blocks after adding $pkg\n";
}
if ($size > $maxdiskblocks) {
print LOG "CD $disknum over-full ($size > $maxdiskblocks). Rollback!\n";
$guess_size = int($hfs_mult * add_packages("--rollback", $cddir, $pkg));
$size=`$size_check $cddir`;
chomp $size;
print LOG "CD $disknum: Real current size is $size blocks after rolling back $pkg\n";
finish_disc($cddir, "");
# Put this package first on the next disc
$overflowpkg = $pkg;
# And reset, to start the next disc
$size = 0;
$disknum++;
} else {
$pkgs_this_cd++;
$pkgs_done++;
$count_since_last_check++;
}
}
close(INLIST);
if ($max_done == 0) {
finish_disc($cddir, " (not)");
}
print LOG "Finished: $pkgs_done packages placed\n";
print "Finished: $pkgs_done packages placed\n";
system("date >> $log");
close(LOG);

10
tools/start_new_disc
View File

@ -81,6 +81,11 @@ if [ ! -d $CDDIR/.disk ] ; then
mkdir $CDDIR/.disk
fi
if [ "$DISC_START_HOOK"x != ""x ] ; then
echo " Calling start hook: $DISC_START_HOOK"
$DISC_START_HOOK $TDIR $MIRROR $DISKNUM $CDDIR "$ARCHES"
fi
echo " Generating the image label and volume id"
echo -n $DISKINFO > $CDDIR/.disk/info
echo -n "$DISKINFO ($DATE)" > $BDIR/$DISKNUM.diskinfo
@ -351,4 +356,9 @@ do
fi
done
if [ "$DISC_PKG_HOOK"x != ""x ] ; then
echo " Calling pkg hook: $DISC_PKG_HOOK"
$DISC_PKG_HOOK $TDIR $MIRROR $DISKNUM $CDDIR "$ARCHES"
fi
exit 0