Update update_popcon to cope better with errors; now uses curl instead of wget

debian/changelog

@@ -11,6 +11,8 @@ debian-cd (3.0.6) UNRELEASED; urgency=low
   [ Steve McIntyre ]
   * Rename list2cds to sort_deps
   * Fix how directory trees are created in update-cd. Closes: #503911
+  * Update update_popcon to cope better with errors; now uses curl instead
+    of wget
 
  -- Frans Pop <fjp@debian.org>  Sun, 02 Nov 2008 00:03:00 +0100
 

debian/control

@@ -11,7 +11,7 @@ Vcs-Browser: http://svn.debian.org/wsvn/debian-cd/trunk?op=log
 
 Package: debian-cd
 Architecture: all
-Depends: perl, cpp, libdigest-md5-perl, tofrodos, apt (>= 0.3.11.1), make, genisoimage, lynx-cur | lynx, grep-dctrl, bc, libcompress-zlib-perl
+Depends: curl, perl, cpp, libdigest-md5-perl, tofrodos, apt (>= 0.3.11.1), make, genisoimage, lynx-cur | lynx, grep-dctrl, bc, libcompress-zlib-perl
 Recommends: hfsutils, netpbm, syslinux-common
 Description: Tools for building (Official) Debian CD set
  Debian-cd is the official tool for building Debian CD set since the potato

tools/update_popcon

@@ -7,8 +7,65 @@
 
 DATE=`date`
 
-OUT=$1
+# Determine temp dir to use
+if [ "$BDIR"x = ""x ] ; then
+    if [ "$TMPDIR"x != ""x ] ; then
+        BDIR=$TMPDIR
+    else
+        BDIR=/tmp
+    fi
+    echo "update_popcon not given a temp dir, using $BDIR" >&2
+fi
+if [ ! -d "$BDIR" ] ; then
+    echo "update_popcon: temp dir '$BDIR' does not exist" >&2
+    exit 1
+fi
 
+# Create temp dir and ensure cleanup
+TDIR=$BDIR/update_popcon.$$
+mkdir -p $TDIR
+trap 'rm -rf $TDIR' EXIT HUP INT QUIT TERM
+
+dl_file () {
+    # Grab the headers for each file so that we can check the size.
+    # We've had problems in the past with download failures causing issues,
+    # so let's make sure that we get the complete file in each case
+    DIST=$1
+    SITE="http://popcon.debian.org/"
+
+    curl -s --head $SITE/$DIST/by_inst > $TDIR/$DIST.head
+    error=$?
+    if [ $error -ne 0 ] ; then
+        echo "Failed to download correctly: curl error $error on HEAD" >&2
+        exit 1
+    fi
+    HEAD_SIZE=`cat $TDIR/$DIST.head | \
+               awk '/^Content-Length/ {gsub("\r","");print $2}'`
+
+    curl -s $SITE/$DIST/by_inst > $TDIR/$DIST
+    error=$?
+    DL_SIZE=`stat -c %s $TDIR/$DIST`
+    if [ $error -ne 0 ] ; then
+        echo "Failed to download correctly: curl error $error" >&2
+        exit 1
+    fi
+
+    if [ $DL_SIZE != $HEAD_SIZE ] ; then
+        echo "Failed to download correctly: expected $HEAD_SIZE bytes, but got $DL_SIZE bytes" >&2
+        exit 1
+    fi
+
+    LASTMOD=`awk '/^Last-Modified:/ {
+                 gsub("\r","")
+                 gsub("^Last-Modified: ","")
+                 print $0}' $TDIR/$DIST.head`
+    echo "$LASTMOD"
+}
+
+MAIN_DATE=`dl_file main`
+CONTRIB_DATE=`dl_file contrib`
+
+OUT=$1
 rm -f $OUT
 
 echo "/*" >> $OUT
@@ -16,11 +73,11 @@ echo "   Popularity Contest results" >> $OUT
 echo "   See the README for details on updating." >> $OUT
 echo "" >> $OUT
 echo "   Last update:               $DATE" >> $OUT
+echo "   Based on main data from    $MAIN_DATE" >> $OUT
+echo "   Based on contrib data from $CONTRIB_DATE" >> $OUT
 echo "*/" >> $OUT
 echo "" >> $OUT
-	wget --output-document - \
-		http://popcon.debian.org/main/by_inst \
-		http://popcon.debian.org/contrib/by_inst | \
-		grep -h '^[^#]'| egrep -v '(Total|-----)' | \
-		sort -rn -k3,3 -k7,7 -k4,4 | grep -v kernel-source | \
-		awk '{print $2}' >> $OUT
+
+cat $TDIR/main $TDIR/contrib | \
+    egrep -v '(^#|Total|-----|kernel-source)' | \
+    sort -rn -k3,3 -k7,7 -k4,4 | awk '{print $2}' >> $OUT