Peppermint
/
debian-cd-clone
mirror of https://salsa.debian.org/images-team/debian-cd.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add initial support for jigdo v2 format 

Add the option to switch to format v2 which uses sha256 everywhere
instead of md5. We can't actually turn this on yet, need to wait for
users to upgrade a bit and get new v2-aware jigdo clients.

But this works as-is for generating v1 until we hit that switch - see
the new JIGOO_CHECKSUM option in CONF.sh
This commit is contained in:
Steve McIntyre 2019-11-21 19:03:11 +00:00
parent 89236b979e
commit 0dd4219af4
7 changed files with 116 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
CONF.sh
View File

@ -181,6 +181,12 @@ export CONTRIB=1
# partition than your source files.
# export COPYLINK=1
# Choose the checksum algorithm used in jigdo and template
# files. Older jigdo tools can only support md5; but we want to move
# to sha256 as a better checksum. Depends on xorriso 1.5.3+ to support
# sha256
export JIGDO_CHECKSUM="md5"
# Options
# export MKISOFS=mkisofs
# export MKISOFS_OPTS="-r" #For normal users

14
Makefile
View File

@ -78,7 +78,7 @@ check_backports_packages=$(BASEDIR)/tools/check_backports_packages
sort_deps=$(BASEDIR)/tools/sort_deps
md5sum=md5sum
jigdo_cleanup=$(BASEDIR)/tools/jigdo_cleanup
grab_md5=$(BASEDIR)/tools/grab_md5
grab_checksums=$(BASEDIR)/tools/grab_checksums
make_image=$(BASEDIR)/tools/make_image
merge_package_lists=$(BASEDIR)/tools/merge_package_lists
update_popcon=$(BASEDIR)/tools/update_popcon
@ -208,7 +208,7 @@ dir-clean:
$(Q)rm -rf $(BDIR)/CD[1234567890]*
$(Q)rm -rf $(TASKDIR)
$(Q)rm -f $(BDIR)/*.filelist*
$(Q)rm -f $(BDIR)/packages-stamp $(BDIR)/upgrade-stamp $(BDIR)/md5-check
$(Q)rm -f $(BDIR)/packages-stamp $(BDIR)/upgrade-stamp $(BDIR)/checksum-check
# Completely cleans the current arch tree
realclean: distclean
@ -488,7 +488,7 @@ image-trees: ok genlist
fi
$(Q)$(BASEDIR)/tools/make_disc_trees.pl $(BASEDIR) $(MIRROR) $(TDIR) $(CODENAME) "$(ARCHES)" "$(MKISOFS)" "$(MKISOFS_OPTS) $(JIGDO_OPTS)"
images: ok $(OUT) $(BDIR)/md5-check
images: ok $(OUT) $(BDIR)/checksum-check
$(Q)$(make_image) "$(BDIR)" "$(ARCHES)" "$(OUT)" "$(DEBVERSION)" "$(MIRROR)" "$(MKISOFS)" "$(MKISOFS_OPTS)" "$(JIGDO_OPTS)" "$(jigdo_cleanup)"
check-number-given:
@ -503,14 +503,14 @@ imagesums:
## MISC TARGETS ##
$(BDIR)/md5-check: mirrorcheck
$(BDIR)/checksum-check: mirrorcheck
mirrorcheck: ok
$(Q)$(grab_md5) $(MIRROR) "$(ARCHES)" $(CODENAME) $(DI_CODENAME) $(BDIR)/md5-check
$(Q)$(grab_checksums) ${JIGDO_CHECKSUM} $(MIRROR) "$(ARCHES)" $(CODENAME) $(DI_CODENAME) $(BDIR)/checksum-check
$(Q)for ARCH in $(ARCHES); do \
if [ -e $(BASEDIR)/data/$(CODENAME)/$$ARCH/extra-sources ]; then \
echo "Extra dedicated source added; need to grab source MD5 info too"; \
$(grab_md5) $(MIRROR) source $(CODENAME) $(DI_CODENAME) $(BDIR)/md5-check; \
echo "Extra dedicated source added; need to grab source checksum info too"; \
$(grab_checksums) $(JIGDO_CHECKSUM) $(MIRROR) source $(CODENAME) $(DI_CODENAME) $(BDIR)/checksum-check; \
fi; \
done

7
debian/changelog vendored
View File

@ -4,6 +4,13 @@ debian-cd (3.1.28) UNRELEASED; urgency=medium
* Add a "with firmware" text to the DISKINFO string when appropriate
Closes: #944207
* Major changes to jigdo support - add support for the new
sha256-based v2 format so we can think about phasing out
dependencies on md5. Will still build using md5 stuff for
compatibility for now - we need a new xorriso release (1.5.3?) to
hit the archive and (more importantly!) users to get current jigdo
clients too before we can switch.
[ Samuel Thibault ]
* Drop support for xorriso versions before 1.2.2
* Add hurd-i386 EFI support.

72
tools/grab_md5 → tools/grab_checksums
View File

@ -1,7 +1,7 @@
#!/bin/sh
#
# grab_md5
# (c) 2004-2016 Steve McIntyre <steve@einval.com>
# grab_checksum
# (c) 2004-2019 Steve McIntyre <steve@einval.com>
#
# GPL v2
#
@ -10,11 +10,25 @@
set -e
MIRROR=$1
ARCHES="$2"
CODENAME=$3
DI_CODENAME=$4
OUT=$5
CHECKSUM=$1
MIRROR=$2
ARCHES="$3"
CODENAME=$4
DI_CODENAME=$5
OUT=$6
case $CHECKSUM in
md5|sha256)
# OK
;;
*)
echo "$0: Unsupported checksum specified: $CHECKSUM"
echo "Abort"
exit 1
;;
esac
export CHECKSUM
for ARCH in $ARCHES
do
@ -34,26 +48,32 @@ do
case $ARCH in
source)
FILES=`find $LOCATIONS -name Sources.gz -o -name Sources.xz`
echo "Using MD5 sums from Sources files:"
echo "Using $CHECKSUM sums from Sources files:"
echo $FILES
$BASEDIR/tools/catz $FILES | MIRROR=$MIRROR perl -ne '
$BASEDIR/tools/catz $FILES | \
MIRROR=$MIRROR CHECKSUM=$CHECKSUM perl -e '
chomp;
my %files;
my $dir;
my $mirror = $ENV{"MIRROR"};
my $filename;
my $mirror = $ENV{"MIRROR"};
my $checksum = $ENV{"CHECKSUM"};
while (<>) {
if (m/^ ([[:xdigit:]]{32}) (\d+) (\S+)/sg) {
$files{$3}{"md5"} = $1;
$files{$3}{"size"} = $2;
}
if (m/^ ([[:xdigit:]]{64}) (\d+) (\S+)/sg) {
$files{$3}{"sha256"} = $1;
$files{$3}{"size"} = $2;
}
if (m/^Directory: (\S+)/sg) {
$dir = $1;
}
if (m/^$/) {
for $filename (keys %files) {
printf("%s %12s %s/%s/%s\n",
$files{$filename}{"md5"},
$files{$filename}{$checksum},
$files{$filename}{"size"},
$mirror, $dir, $filename);
}
@ -64,19 +84,21 @@ do
*)
FILES=`find $LOCATIONS -name Packages.gz \
-o -name Packages.xz | grep binary-$ARCH`
echo "Using MD5 sums from Packages files:"
echo "Using $CHECKSUM sums from Packages files:"
echo $FILES
NUM_FILES=`echo $FILES| wc -w`
if [ $NUM_FILES -eq 1 ] ; then
echo "No files found for arch $ARCH. Abort!"
exit 1
fi
$BASEDIR/tools/catz $FILES | MIRROR=$MIRROR perl -ne '
$BASEDIR/tools/catz $FILES | \
MIRROR=$MIRROR CHECKSUM=$CHECKSUM perl -e '
chomp;
my $mirror = $ENV{"MIRROR"};
my $checksum = $ENV{"CHECKSUM"};
my $filename;
my $size;
my $md5;
my $sum;
while (<>) {
if (m/^Filename: (\S+)/sg) {
$filename = $1;
@ -84,11 +106,14 @@ do
if (m/^Size: (\S+)/sg) {
$size = $1;
}
if (m/^MD5sum: (\S+)/sg) {
$md5 = $1;
if ($checksum eq "md5" and (m/^MD5sum: (\S+)/sg)) {
$sum = $1;
} elsif ($checksum eq "sha256" and (m/^SHA256: (\S+)/sg)) {
$sum = $1;
}
if (m/^$/) {
printf("%s %12s %s/%s\n", $md5, $size, $mirror, $filename);
printf("%s %12s %s/%s\n", $sum, $size, $mirror, $filename);
$sum = "";
}
}' | sort | uniq >> $OUT
# Use the new D-I images. Do NOT use the "current"
@ -96,15 +121,19 @@ do
for VER in $MIRROR/dists/$DI_CODENAME/main/installer-$ARCH/*
do
if [ -d $VER ] && [ ! -L $VER ] ; then
FILE=$VER/images/MD5SUMS
echo "Using MD5 sums from d-i: $FILE"
if [ $CHECKSUM = md5 ]; then
FILE=$VER/images/MD5SUMS
else
FILE=$VER/images/SHA256SUMS
fi
echo "Using $CHECKSUM sums from d-i: $FILE"
LOC=dists/$DI_CODENAME/main/installer-$ARCH/`basename $VER`/images
for ENTRY in `cat $FILE | sed 's/ /:/g'`
do
PATH=`echo $ENTRY | /bin/sed "s?^.*:\./?$MIRROR/$LOC/?g"`
MD5=`echo $ENTRY | /bin/sed 's/:.*$//g'`
CSUM=`echo $ENTRY | /bin/sed 's/:.*$//g'`
SIZE=`/usr/bin/stat -c %s $PATH`
printf '%s %12.12s %s\n' $MD5 $SIZE $PATH
printf '%s %12.12s %s\n' $CSUM $SIZE $PATH
done | sort | uniq >> $OUT
fi
done
@ -113,3 +142,4 @@ do
done
exit 0

5
tools/make_disc_trees.pl
View File

@ -203,9 +203,8 @@ while (defined (my $pkg = <INLIST>)) {
$mkisofs_opts = "";
}
if ($disknum <= $maxjigdos) {
$mkisofs_opts = "$mkisofs_opts -jigdo-jigdo /dev/null";
$mkisofs_opts = "$mkisofs_opts -jigdo-template /dev/null";
$mkisofs_opts = "$mkisofs_opts -md5-list /dev/null";
# Set things to /dev/null - we're only doing a
# sizing run here
$mkisofs_opts = "$mkisofs_opts -o /dev/null";
}
if ( -e "$bdir/$disknum.mkisofs_dirs" ) {

49
tools/make_image
View File

@ -42,6 +42,15 @@ last_minute_update () {
done
}
# Grab the xorriso version and turn it into a number we can use
xorriso_version() {
$MKISOFS --version 2>&1 | awk '
/^xorriso version/ {
split($4, ver, ".")
print ver[1]*10000+ver[2]*100+ver[3]
}'
}
for file in $FILES
do
dir=${file%%.volid}
@ -82,8 +91,36 @@ do
JIGDO_WANTED=0
fi
# Work out common jigdo options if needed
if [ $JIGDO_WANTED = 1 ]; then
XOR_VER=$(xorriso_version)
if [ $XOR_VER -lt 10502 ]; then
# Only supports jigdo format v1
case "$JIGDO_CHECKSUM"x in
"md5"x|x)
JIGDO_CHECKSUM_OPTS="-md5-list $DIR/checksum-check \
-jigdo-force-md5 /pool/"
;;
*)
echo "Older xorriso version $XOR_VER does not support jigdo checksum algorithm $JIGDO_CHECKSUM"
exit 1
;;
esac
else
JIGDO_CHECKSUM_OPTS="-checksum-list $DIR/checksum-check \
-jigdo-checksum-algorithm $JIGDO_CHECKSUM \
-jigdo-force-checksum /pool/"
fi
JIGDO_OPTS="$JIGDO_CHECKSUM_OPTS $JIGDO_OPTS \
-jigdo-jigdo $OUT/$OUTFILE.jigdo \
-jigdo-template $OUT/$OUTFILE.template \
-jigdo-map Debian=$MIRROR/ \
-jigdo-exclude boot$n"
fi
# Actually make the ISO/jigdo images. Long command lines
# here... :-(
# here, with lots of stuff included :-(
CMD=
if [ $JIGDO_WANTED = 0 ] && [ $ISO_WANTED = 1 ] ; then
@ -94,22 +131,12 @@ do
CMD="$MKISOFS $MKISOFS_OPTS -V '$volid' \
-o $OUT/$OUTFILE.iso \
-jigdo-jigdo $OUT/$OUTFILE.jigdo \
-jigdo-template $OUT/$OUTFILE.template \
-jigdo-map Debian=$MIRROR/ \
-jigdo-exclude boot$n \
-md5-list $DIR/md5-check \
$JIGDO_OPTS $opts $dirs CD$n"
elif [ $JIGDO_WANTED = 1 ] && [ $ISO_WANTED = 0 ] ; then
CMD="$MKISOFS $MKISOFS_OPTS -V '$volid' \
-o /dev/null \
-jigdo-jigdo $OUT/$OUTFILE.jigdo \
-jigdo-template $OUT/$OUTFILE.template \
-jigdo-map Debian=$MIRROR/ \
-jigdo-exclude boot$n \
-md5-list $DIR/md5-check \
$JIGDO_OPTS $opts $dirs CD$n"
else

8
update-cd
View File

@ -45,6 +45,8 @@ OUT=/srv/cdbuilder.debian.org/src/deb-cd/tmp/stretch-update/cd-out
# Location(s) for the snapshot(s)
SNAPSHOT='Debian=https://us.cdimage.debian.org/cdimage/snapshot/Debian/ --try-last'
JIGOO_CHECKSUM=md5
# Type of disc image, cd or dvd (lower case)
if [ "$1"x = ""x ] ; then
TYPE=cd
@ -81,7 +83,7 @@ DATE=`date +%Y%m%d`
BASEDIR=`pwd`
SECTS="main non-free contrib"
VERBOSE=2
GRAB_MD5=$BASEDIR/tools/grab_md5
GRAB_CHECKSUMS=$BASEDIR/tools/grab_checksums
export VERBOSE BASEDIR SECTS
FIRSTVER=`echo $VER | cut -c 1-4`0
@ -164,7 +166,7 @@ make_cd () {
-jigdo-template ${JIGDODIR}/${BASENAME}.template \
-jigdo-map Debian=$MIRROR/ \
-jigdo-force-md5 /pool/ \
-md5-list $UPD/md5-check \
-checksum-list $UPD/checksum-check \
$UPD/CD$CDNUM 2>&1 | grep "extents written"
# $UPD/CD$CDNUM
if [ $? -ne 0 ] ; then
@ -280,7 +282,7 @@ if [ "$NONFREE"x != "1"x ] ; then
mv -f $UPD/list1 $UPD/list
fi
$GRAB_MD5 $MIRROR_NORM "$ARCHLIST all" $CODENAME $CODENAME $UPD/md5-check
$GRAB_CHECKSUMS $JIGOO_CHECKSUM $MIRROR_NORM "$ARCHLIST all" $CODENAME $CODENAME $UPD/checksum-check
for THISARCH in $ARCHLIST
do