Peppermint
/
debian-cd-clone
mirror of https://salsa.debian.org/images-team/debian-cd.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Extend the code that copes with too-large binary packages to cope with 

too-large source packages too.
This commit is contained in:
Steve McIntyre 2009-12-29 01:27:25 +00:00
parent e355a14642
commit acf3d69328
6 changed files with 87 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
debian/changelog vendored
View File

@ -47,9 +47,9 @@ debian-cd (3.1.3) UNRELEASED; urgency=low
Mark Post and Bastian Blank for providing the pointers to the missing bits.
[ Steve McIntyre ]
* Add support for coping with packages that are too large to fit. If we
have to remove packages and their dependencies, list them in
README.excluded in the root of disc 1.
* Add support for coping with binary and source packages that are too
large to fit. If we have to remove packages and their dependencies,
list them in README.excluded in the root of disc 1.
-- Frans Pop <fjp@debian.org> Sun, 23 Aug 2009 08:59:25 +0200

30
tools/grab_source_list
View File

@ -6,7 +6,7 @@ IN="$3"
OUT="$4"
echo "DONEDONE" > $BDIR/DONE
awk '
awk -v max_size=$MAX_PKG_SIZE '
/^Package:/ {
srcname=$2
srcs_done++
@ -32,19 +32,39 @@ awk '
next
}
/^Files:/ {
in_files = 1
next
}
/^ / {
if (in_files) {
size[srcname]+=$2
next
}
}
/.*/ {
in_files = 0
if (parsed) {
printf("source:%s:%s\n", component[$0], $0)
if (size[$0] > max_size) {
printf("source:%s:%s-SRCTOOBIG\n", component[$0], $0)
} else {
printf("source:%s:%s\n", component[$0], $0)
}
}
}
' $ADIR/$CODENAME-source/apt-state/lists/*Sources $BDIR/DONE $IN > $BDIR/list.mid
if [ "$NONFREE"x = "1"x ] ; then
cat $BDIR/list.mid > $OUT
grep -v SRCTOOBIG $BDIR/list.mid > $OUT
else
grep -v :non-free: $BDIR/list.mid > $OUT
grep -v -e :non-free: -e SRCTOOBIG $BDIR/list.mid > $OUT
fi
if [ "$EXTRANONFREE"x = "1"x ] ; then
grep :non-free: $BDIR/list.mid >> $OUT
grep :non-free: $BDIR/list.mid | grep -v SRCTOOBIG >> $OUT
fi
awk -F : '
/SRCTOOBIG/ { print $3 }
' $BDIR/list.mid > $BDIR/sourcetoobig

10
tools/make_disc_trees.pl
View File

@ -213,7 +213,7 @@ while (defined (my $pkg = <INLIST>)) {
# lists intersect and we should re-include some packages
if (scalar @unexclude_packages && scalar @excluded_package_list) {
foreach my $reinclude_pkg (@excluded_package_list) {
my ($arch, $component, $pkgname) = split /:/, $reinclude_pkg;
my ($arch, $component, $pkgname, $pkgsize) = split /:/, $reinclude_pkg;
foreach my $entry (@unexclude_packages) {
if (($pkgname =~ /^\Q$entry\E$/m)) {
print LOG "Re-including $reinclude_pkg due to match on \"\^$entry\$\"\n";
@ -316,7 +316,7 @@ sub load_packages_cache {
while (defined (my $pkg = <INLIST>)) {
chomp $pkg;
my ($junk, $component, $pkgname) = split /:/, $pkg;
my ($junk, $component, $pkgname, $pkgsize) = split /:/, $pkg;
push @pkglist, $pkgname;
}
close INLIST;
@ -347,7 +347,7 @@ sub load_packages_cache {
sub should_start_extra_nonfree {
my $pkg = shift;
my ($arch, $component, $pkgname) = split /:/, $pkg;
my ($arch, $component, $pkgname, $pkgsize) = split /:/, $pkg;
if ( ($component eq "non-free") && $extranonfree) {
$extranonfree = 0; # Flag that we don't need to start new next time!
@ -359,7 +359,7 @@ sub should_start_extra_nonfree {
sub should_exclude_package {
my $pkg = shift;
my ($arch, $component, $pkgname) = split /:/, $pkg;
my ($arch, $component, $pkgname, $pkgsize) = split /:/, $pkg;
my $should_exclude = 0;
foreach my $entry (@exclude_packages) {
@ -991,7 +991,7 @@ sub add_packages {
}
my $pkg = shift;
my ($arch, $component, $pkgname) = split /:/, $pkg;
my ($arch, $component, $pkgname, $pkgsize) = split /:/, $pkg;
if ("$arch" eq "" or "$pkgname" eq "" or "$pkgname" eq "") {
die "inconsistent data passed to add_packages: $pkg\n";

39
tools/merge_package_lists
View File

@ -59,7 +59,7 @@ done
echo "# Starting merge at "`date` >> $BDIR/list.mid
if [ $source = "yes" ] ; then
echo "DONEDONE" > $BDIR/DONE
awk '
awk -v max_size=$MAX_PKG_SIZE '
# First parse the Sources files
/^Package:/ {
@ -77,6 +77,17 @@ if [ $source = "yes" ] ; then
}
}
/^Files:/ {
in_files = 1
next
}
/^ / {
if (in_files) {
size[srcname]+=$2
next
}
}
# Done reading the Sources
/^DONEDONE/ {
parsed=1
@ -86,11 +97,13 @@ if [ $source = "yes" ] ; then
# Now start placing source packages, depending on the order of the binary packages
/.*/ {
in_files = 0
if (parsed) {
split($0,fields,":")
arch=fields[1]
component=fields[2]
pkg=fields[3]
pkgsize=fields[4]
source=bin2src[pkg]
if ("" == source) {
if ("main-installer" == component) {
@ -123,13 +136,21 @@ if [ $source = "yes" ] ; then
}
}
}
printf("%d:%s\n", FNR, $0)
if (size[source] <= max_size) {
printf("%d:%s\n", FNR, $0)
} else {
printf("%d:%s-DEPTOOBIG\n", FNR, $0)
}
}
}
END {
for (source in included) {
printf("%d:aaaaaaaaaaaaaaaaaa:%s:%s\n", indexnr[source], sourcecomp[source], source)
if (size[source] > max_size) {
printf("%d:aaaaaaaaaaaaaaaaaa:%s:%s-SRCTOOBIG\n", indexnr[source], sourcecomp[source], source, size[source])
} else {
printf("%d:aaaaaaaaaaaaaaaaaa:%s:%s\n", indexnr[source], sourcecomp[source], source, size[source])
}
}
}' $ADIR/$CODENAME-source/apt-state/lists/*Sources $BDIR/DONE $list | sort -nk 1 -t : >> $BDIR/list.mid
else
@ -141,7 +162,7 @@ echo "# Ending merge pass 1 at "`date` >> $BDIR/list.mid
# prepended to allow us to shuffle them properly. Remove any comments
# logged from the awk code above, then sort and remove the line numbers
# again
grep -v ^# $BDIR/list.mid | awk -F : '
grep -v -e ^# -e TOOBIG $BDIR/list.mid | awk -F : '
{
if (!($4 in add))
{
@ -149,7 +170,7 @@ grep -v ^# $BDIR/list.mid | awk -F : '
}
gsub("aaaaaaaaaaaaaaaaaa", "source", $2) # Undo the source sorting hack
gsub("main-installer", "main", $3)
add[$4] = add[$4] $2 ":" $3 ":" $4 " "
add[$4] = add[$4] $2 ":" $3 ":" $4 ":" $5 " "
}
END {
for (i = 0; i < number_packages; i++)
@ -160,3 +181,11 @@ END {
}
}' > $OUT
echo "# Ending merge pass 2 at "`date` >> $BDIR/list.mid
awk -F : '
/SRCTOOBIG/ { print $3 }
' $BDIR/list.mid > $BDIR/sourcetoobig
awk -F : '
/DEPTOOBIG/ { print $3 }
' $BDIR/list.mid > $BDIR/sourcetoobig-deps

3
tools/sort_deps
View File

@ -294,7 +294,8 @@ open(CDLIST, "> $dir/packages.$arch")
|| die "Can't write in $dir/$_.packages.$arch: $!\n";
foreach (@output) {
my $component = $packages{$_}{"Component"};
print CDLIST "$arch:$component:$_\n";
my $size = $packages{$_}{"Size"};
print CDLIST "$arch:$component:$_:$size\n";
$count++;
}
close CDLIST;

20
tools/start_new_disc
View File

@ -255,10 +255,26 @@ if [ $DISKNUM = 1 ] ; then
done
# See if we had to remove anything because it was too big. Warn the user.
if [ -s $BDIR/sourcetoobig ] ; then
echo " Adding README.excluded (some source files were too big)"
echo "For size reasons, the following source packages were excluded from this disc set:" >> $CDDIR/README.excluded
echo >> $CDDIR/README.excluded
cat $BDIR/sourcetoobig >> $CDDIR/README.excluded
if [ -s $BDIR/sourcetoobig-deps ] ; then
echo "and that caused the following packages to be also excluded because" >> $CDDIR/README.excluded
echo "of dependencies:" >> $CDDIR/README.excluded
echo >> $CDDIR/README.excluded
cat $BDIR/sourcetoobig-deps >> $CDDIR/README.excluded
echo >> $CDDIR/README.excluded
echo "Beware that this may break dependencies in this disc set." >> $CDDIR/README.excluded
fi
echo >> $CDDIR/README.excluded
fi
grep "refused because of toobig" $BDIR/log.sort_deps* | sort -u | awk '{print $1}' > $BDIR/toobig
if [ -s $BDIR/toobig ] ; then
echo " Adding README.excluded (some files were too big)"
echo "For size reasons, the following packages were excluded from this disc set:" > $CDDIR/README.excluded
echo " Adding README.excluded (some package files were too big)"
echo "For size reasons, the following packages were excluded from this disc set:" >> $CDDIR/README.excluded
echo >> $CDDIR/README.excluded
cat $BDIR/toobig >> $CDDIR/README.excluded
grep "Can't add.*toobig" $BDIR/log.sort_deps* | sort -u | awk '{print $3}' > $BDIR/toobig-deps