debian-cd-clone/contrib/cronjob.weekly

#!/bin/bash

TOPDIR=$(dirname $0)
if [ "$TOPDIR" = "." ] ; then
    TOPDIR=`pwd`
fi
export TOPDIR

. $TOPDIR/settings.sh

BUILDLOCK=$HOME/.debian-cd.lock

export PUBDIRJIG=$PUBDIR/weekly-builds
export PUBDIROSTRACE=${OUT_BASE}/.live/trace-os
export PUBDIRLIVETRACE=${OUT_BASE}/.live/trace-live
export DATE_BUILD="$DATE-$BUILDNUM"
export CD_I_F="${OUT_BASE}/unofficial/non-free/cd-including-firmware"

. $TOPDIR/common.sh

# Make sure the machine isn't rebooted while we're busy
reboot_lock

# Uncomment the following to use daily d-i builds for these images
# rather than what's in the archive
USE_DAILY_DI=Y

if [ "$RELEASE_BUILD"x = ""x ] ; then
    export RSYNC_TARGET=${OUT_BASE}/weekly-builds
    export RSYNC_TARGET_FIRMWARE=${CD_I_F}/weekly-builds
else
    export RSYNC_TARGET=${OUT_BASE}/.${RELEASE_BUILD}/debian-cd
    export RSYNC_TARGET_FIRMWARE=${CD_I_F}/.${RELEASE_BUILD}/debian-cd
    export CONF=~/build.${CODENAME}/CONF.sh.${RELEASE_BUILD}
    export RELEASE_BUILD=$RELEASE_BUILD
    # If we're doing a normal set of daily/weekly builds, leave the
    # checksum filenames alone. Otherwise, make life easier for people
    # combining things later and append a suitable name as we build.
    export EXTENSION=".large"
    USE_DAILY_DI=N
    NOSNAP=1
fi

. images4testing_d-i

if [ "$USE_DAILY_DI"x = "Y"x ] ; then
    export DI=sid
    export DI_WWW_HOME=default
    export DI_DIR="$ARCH_DI_DIR"
else
    unset DI_WWW_HOME
fi

export DI_DIST
export DI_CODENAME
export OUT_FREE_SID=${PUBDIRJIG}
export OUT_FREE_TST=${PUBDIRJIG}
export OUT_FW_SID=${PUBDIRJIG}-firmware
export OUT_FW_TST=${PUBDIRJIG}-firmware

BUILDS_RUNNING=""

#export NOEDU=1
#export NOOPENSTACK=1
#export NOLIVE=1

if lockfile -r0 $BUILDLOCK ; then
#    echo "NOT checking for git updates"
    rm -f $PUBDIRLIVETRACE $PUBDIROSTRACE
    ~/build.${CODENAME}/cronjob.weekly-live &

    echo "git update debian-cd"
    cd debian-cd && git pull ; cd ..

    mkdir -p ${PUBDIRJIG}/trace
    mkdir -p $RSYNC_TARGET/trace $RSYNC_TARGET_FIRMWARE/trace

    # Keep track of the serial for the archive we're building against,
    # for later archive diffing for release announce preparation:
    serial=$(get_archive_serial)
    echo "$serial" > $RSYNC_TARGET/trace/archive-serial
    echo "$serial" > $RSYNC_TARGET_FIRMWARE/trace/archive-serial

    # Work out the default desktop, and do *not* build a CD1 for that
    # desktop - it'll be done in the full set anyway
    TASKSEL_DEB=$(ARCHES=amd64 ./debian-cd/tools/which_deb ${MIRROR} ${DI_CODENAME} task-desktop binary)
    DEFAULT_DESKTOP=$(dpkg --info ${MIRROR}/${TASKSEL_DEB} |
	awk '/Recommends: task-.*-desktop/ {split($2,a,"-"); print toupper(a[2])}')
    echo default desktop is $DEFAULT_DESKTOP, drop CD1 for that desktop
    export NO${DEFAULT_DESKTOP}CD=1

#    echo "NOT checking for popcon updates"
    if [ "$NOPOPCON"x = ""x ] ; then
	cd debian-cd && ./tools/update_popcon tasks/${CODENAME}/popularity-contest
	error=$?
	if [ $error != 0 ]; then
	    echo "popcon update failed, error $error";
	    echo "aborting";
	    exit 1
	fi
	cd ..
    fi

    cd $TOPDIR
    for arch in $ARCHES; do
        # Reset envvars for next iteration
        arch_error=""
        arch_start=`now`

        rm -rf ${PUBDIRJIG}/$arch ${PUBDIRJIG}-firmware/$arch
        for i in iso-cd jigdo-cd; do
            mkdir -p ${PUBDIRJIG}/$arch/$i
            if $(arch_has_firmware $arch) ; then
		mkdir -p ${PUBDIRJIG}-firmware/$arch/$i
	    fi
        done

        if [ "$arch" != multi-arch ] ; then
#            export DI_DIR="$ARCH_DI_DIR" 
	    export VARIANTS=""

            # Full DVD set
            if [ "$NODVD"x = ""x ] ; then
                export MAX_PKG_SIZE=999999999999
                case $arch in
                    i386|amd64)
                        export MAXISOS=3
			export MAXJIGDOS=ALL
			export VARIANTS=xen
                        # Special case: make DVD1 fit on a 4GB USB
                        # stick (#612074)
			export FORCE_CD_SIZE1=STICK4GB
			;;
                    source)
                        export MAXISOS=ALL; export MAXJIGDOS=ALL ;;
                    powerpc|arm64)
                        # Special cases: make DVD1 fit on a 4GB USB
                        # stick
			export FORCE_CD_SIZE1=STICK4GB
                        export MAXISOS=1; export MAXJIGDOS=ALL ;;
                    *)
                        export MAXISOS=1; export MAXJIGDOS=ALL ;;
                esac
                export DESKTOP=all
		build_started DVD
                INSTALLER_CD=3 TASK=Debian-all \
                    KERNEL_PARAMS='desktop=all' \
                    ./testingcds "$arch" &

		# Single DVD with firmware
		if $(arch_has_firmware $arch) ; then
                    case $arch in
			i386|amd64)
			    export VARIANTS=xen
			    ;;
		    esac
                    build_started DVDFIRMWARE
                    COMPLETE=0 INSTALLER_CD=I \
                            FORCE_FIRMWARE=1 CDNAME=firmware \
                            KERNEL_PARAMS="desktop=all" \
                            DESKTOP=all \
                            TASK=Debian-all \
			    MAXISOS=1 MAXJIGDOS=1 \
                            DI=${DI} DI_DIST=${DI} \
                            ./testingcds "$arch" &
                fi
		unset FORCE_CD_SIZE1
            fi

	    # Full BD set
            if [ "$NOBD"x = ""x ] ; then
		unset DESKTOP
                export MAX_PKG_SIZE=999999999999
                case $arch in
                    i386|amd64|source)
                        export MAXISOS=0
                        export MAXJIGDOS=ALL
                        export DESKTOP=all
			export VARIANTS=xen
			build_started BD
                        INSTALLER_CD=9 TASK=Debian-all \
                            KERNEL_PARAMS='desktop=all' \
                            ./testingcds "$arch" &
                        ;;
                    *)
                        echo "  Not running BD build for $arch"
                        ;;
                esac
            fi

            # Full DLBD set
            if [ "$NOBD"x = ""x ] && [ "$NODLBD"x = ""x ] ; then
		unset DESKTOP
                export MAX_PKG_SIZE=999999999999
                case $arch in
                    i386|amd64|source)
                        export MAXISOS=0
                        export MAXJIGDOS=ALL
                        export DESKTOP=all
			export VARIANTS=xen
			build_started DLBD
                        INSTALLER_CD=D TASK=Debian-all \
                            KERNEL_PARAMS='desktop=all' \
                            ./testingcds "$arch" &
                        ;;
                    *)
                        echo "  Not running DLBD build for $arch"
                        ;;
                esac
            fi

            # 16G USB image
            if [ "$NOUSB"x = ""x ] ; then
		unset DESKTOP
                export MAX_PKG_SIZE=999999999999
                case $arch in
                    i386|amd64)
                        export MAXISOS=0
                        export MAXJIGDOS=1
                        export DESKTOP=all
			export VARIANTS=xen
			build_started USB16G
                        INSTALLER_CD=M TASK=Debian-all \
                            KERNEL_PARAMS='desktop=all' \
                            ./testingcds "$arch" &
                        ;;
                    *)
                        echo "  Not running USB16G build for $arch"
                        ;;
                esac
            fi

            # individual desktop cd #1: xfce only these days
            if [ "$arch"x != "source"x ] && [ "$NOCD"x = ""x ] ; then
                if [ "$NOXFCECD"x = ""x ] ; then
                    export MAX_PKG_SIZE=300000000
                    export DESKTOP=xfce
                    export MAXISOS=1; export MAXJIGDOS=1
                    export VARIANTS=""
                    build_started XFCECD
                    INSTALLER_CD=5 TASK=Debian-xfce \
                        KERNEL_PARAMS='desktop=xfce' \
                        OMIT_RELEASE_NOTES=1 OMIT_MANUAL=1 \
                        MAXCDS=1 MAXISOS=ALL MAXJIGDOS=ALL \
                        ./testingcds "$arch" &
                fi
            fi

            # netinst CD
            if [ "$arch"x != "source"x ] && [ "$NOCD"x = ""x ] && [ "$NONI"x = ""x ] ; then
                export MAX_PKG_SIZE=300000000
                export DESKTOP=all
                export MAXISOS=1; export MAXJIGDOS=1
                export VARIANTS=""

                build_started NI
                OMIT_RELEASE_NOTES=1 OMIT_MANUAL=1 NORECOMMENDS=1 \
                    NOSUGGESTS=1 COMPLETE=0 INSTALLER_CD=2 \
                    KERNEL_PARAMS="desktop=all" \
                    DESKTOP=all VARIANTS=xen \
                    TASK=debian-installer+kernel MAXISOS=ALL MAXJIGDOS=ALL \
                    DI=${DI} DI_DIST=${DI} \
                    ./testingcds "$arch" &		

		if $(arch_has_firmware $arch) ; then
                    build_started NIFIRMWARE
                    OMIT_RELEASE_NOTES=1 OMIT_MANUAL=1 NORECOMMENDS=1 \
                        NOSUGGESTS=1 COMPLETE=0 INSTALLER_CD=C \
                        FORCE_FIRMWARE=1 CDNAME=firmware \
                        KERNEL_PARAMS="desktop=all" \
                        DESKTOP=all VARIANTS=xen \
                        TASK=debian-installer+kernel MAXISOS=ALL MAXJIGDOS=ALL \
                        DI=${DI} DI_DIST=${DI} \
                        ./testingcds "$arch" &
                fi

		# Special Mac-only builds
		if [ "$arch" = "i386" ] || [ "$arch" = "amd64" ] ; then
                    build_started MACNI
                    OMIT_RELEASE_NOTES=1 OMIT_MANUAL=1 NORECOMMENDS=1 \
                        NOSUGGESTS=1 COMPLETE=0 INSTALLER_CD=F \
                        BOOT_METHODS=BIOS CDNAME="debian-mac" \
                        KERNEL_PARAMS="desktop=all" \
                        DESKTOP=all VARIANTS=xen \
                        TASK=debian-installer+kernel MAXISOS=ALL MAXJIGDOS=ALL \
                        DI=${DI} DI_DIST=${DI} \
                        ./testingcds "$arch" &
                fi

		# Special debian-edu builds, only minor tweaks. #846006
		if [ "$NOEDU"x = ""x ] ; then
		    if [ "$arch" = "i386" ] || [ "$arch" = "amd64" ] ; then

			# netinst
			build_started EDUNI
			OMIT_RELEASE_NOTES=1 OMIT_MANUAL=1 NORECOMMENDS=1 \
					  NOSUGGESTS=1 COMPLETE=0 INSTALLER_CD=G \
					  CDNAME=debian-edu \
					  DEBIAN_EDU=1 \
					  KERNEL_PARAMS="modules=debian-edu-install-udeb" \
					  DESKTOP=xfce VARIANTS=xen \
					  EXCLUDE=debian-edu-exclude \
					  UDEB_EXCLUDE="$BASEDIR"/data/$CODENAME/udeb_exclude \
					  TASK=Debian-edu-netinst \
					  MAXISOS=1 MAXJIGDOS=1 \
					  DI=${DI} DI_DIST=${DI} \
					  ./testingcds "$arch" &

			# netinst with firmware
			build_started EDUNIFIRMWARE
			OMIT_RELEASE_NOTES=1 OMIT_MANUAL=1 NORECOMMENDS=1 \
					  NOSUGGESTS=1 COMPLETE=0 INSTALLER_CD=H \
					  FORCE_FIRMWARE=1 \
					  CDNAME=firmware-edu \
					  DEBIAN_EDU=1 \
					  KERNEL_PARAMS="modules=debian-edu-install-udeb" \
					  DESKTOP=xfce VARIANTS=xen \
					  EXCLUDE=debian-edu-exclude \
					  UDEB_EXCLUDE="$BASEDIR"/data/$CODENAME/udeb_exclude \
					  TASK=Debian-edu-netinst \
					  MAXISOS=1 MAXJIGDOS=1 \
					  DI=${DI} DI_DIST=${DI} \
					  ./testingcds "$arch" &

			if [ "$NOUSB"x = ""x ] ; then
			    # USB "BD" image
			    build_started EDUUSB
			    NORECOMMENDS=0 COMPLETE=0 INSTALLER_CD=K \
				    CDNAME=debian-edu \
				    DEBIAN_EDU=1 \
				    KERNEL_PARAMS="modules=debian-edu-install-udeb" \
				    DESKTOP=xfce VARIANTS=xen \
				    EXCLUDE=debian-edu-exclude \
				    TASK=Debian-edu-full \
				    MAXISOS=1 MAXJIGDOS=1 \
				    DI=${DI} DI_DIST=${DI} \
				    ./testingcds "$arch" &

			    # USB "BD" image with firmware
			    build_started EDUUSBFIRMWARE
			    NORECOMMENDS=0 COMPLETE=0 INSTALLER_CD=L \
				    FORCE_FIRMWARE=1 \
				    CDNAME=firmware-edu \
				    DEBIAN_EDU=1 \
				    KERNEL_PARAMS="modules=debian-edu-install-udeb" \
				    DESKTOP=xfce VARIANTS=xen \
				    EXCLUDE=debian-edu-exclude \
				    TASK=Debian-edu-full \
				    MAXISOS=1 MAXJIGDOS=1 \
				    DI=${DI} DI_DIST=${DI} \
				    ./testingcds "$arch" &
			fi
		    fi
		fi
            fi

	    # We've spawned a load of builds; wait here for them all
	    # to finish
	    catch_parallel_builds

        else # multi-arch

	    if [ "$NOCD"x = ""x ] && [ "$NONI"x = ""x ] ; then
                export DESKTOP=all
                build_started NI
                OMIT_RELEASE_NOTES=1 OMIT_MANUAL=1 NORECOMMENDS=1 \
                    NOSUGGESTS=1 COMPLETE=0 INSTALLER_CD=2 \
                    KERNEL_PARAMS="desktop=all" \
                    DESKTOP=all \
		    EXCLUDE_686_PAE_KERNEL=1 \
                    TASK=debian-installer+kernel \
                    MAXISOS=ALL MAXJIGDOS=ALL \
                    DI=${DI} DI_DIST=${DI} \
                    ./testingcds "amd64 i386" &

		if $(arch_has_firmware $arch) ; then
                    build_started NIFIRMWARE
		    export FORCE_CD_SIZE1=CD700
                    OMIT_RELEASE_NOTES=1 OMIT_MANUAL=1 NORECOMMENDS=1 \
                        NOSUGGESTS=1 COMPLETE=0 INSTALLER_CD=C \
                        FORCE_FIRMWARE=1 CDNAME=firmware \
                        KERNEL_PARAMS="desktop=all" \
                        DESKTOP=all \
			EXCLUDE_686_PAE_KERNEL=1 \
                        TASK=debian-installer+kernel \
                        MAXISOS=ALL MAXJIGDOS=ALL \
                        DI=${DI} DI_DIST=${DI} \
                        ./testingcds "amd64 i386" &
		    unset FORCE_CD_SIZE1
                fi

	    fi

	    # We've spawned a load of builds; wait here for them all
	    # to finish
	    catch_parallel_builds

        fi # end of multi-arch

        if [ "$RELEASE_BUILD"x = ""x ] ; then
	    echo "$arch: Not generating torrent files for a non-release build"
	else
	    echo "$arch: Generating torrent files"
	    ~/build.${CODENAME}/mktorrent ${PUBDIRJIG}/$arch/iso-*/*.iso
	    if [ -d ${PUBDIRJIG}-firmware/$arch ] ; then
		~/build.${CODENAME}/mktorrent ${PUBDIRJIG}-firmware/$arch/iso-*/*.iso
	    fi
	fi

	echo "$arch: Generating checksum files"
        for dir in ${PUBDIRJIG}/$arch/jigdo-*; do
            generate_checksums_for_arch $arch $dir
        done
        if [ -d ${PUBDIRJIG}-firmware/$arch ] ; then
            for dir in ${PUBDIRJIG}-firmware/$arch/jigdo-*; do
                generate_checksums_for_arch $arch $dir
            done
        fi

        if [ "$NOSYNC"x = ""x ] ; then
            if [ "$arch_error"x = "none"x ] || [ "$arch_error"x = ""x ] ; then
                if [ "$RELEASE_BUILD"x = ""x ] ; then
                    echo "Signing checksums files using the automatic key"
                    ~/build.${CODENAME}/sign-images ${PUBDIRJIG} $arch
                fi
                echo "  Running ~/build.${CODENAME}/iso_run ${PUBDIRJIG}/ $RSYNC_TARGET/ $arch \"\" &"
                ~/build.${CODENAME}/iso_run ${PUBDIRJIG}/ $RSYNC_TARGET/ $arch "" &
                if [ -d ${PUBDIRJIG}-firmware/$arch ] ; then
                    if [ "$RELEASE_BUILD"x = ""x ] ; then
                        echo "Signing checksums files using the automatic key"
                        ~/build.${CODENAME}/sign-images ${PUBDIRJIG}-firmware $arch
                    fi
                    echo "  Running ~/build.${CODENAME}/iso_run ${PUBDIRJIG}-firmware/ $RSYNC_TARGET_FIRMWARE/ $arch firmware &"
                    ~/build.${CODENAME}/iso_run ${PUBDIRJIG}-firmware/ $RSYNC_TARGET_FIRMWARE/ $arch firmware &
                fi
            else
                ~/build.${CODENAME}/report_build_error ${PUBDIRJIG} $RSYNC_TARGET $arch "$arch_error"
                if [ -d ${PUBDIRJIG}-firmware/$arch ] ; then
		    ~/build.${CODENAME}/report_build_error ${PUBDIRJIG}-firmware $RSYNC_TARGET_FIRMWARE/ $arch "$arch_error"
		fi
            fi
        fi

    done

    if [ "$NOFW"x = ""x ] ; then
	~/build.${CODENAME}/generate_firmware_images ${CODENAME}
	~/build.${CODENAME}/generate_firmware_images sid
    fi
    
    if [ $(hostname) = casulana ]; then
	rsync_to_pettersson ${OUT_BASE}/cd-sources/ cd-sources/
	publish_on_pettersson cd-sources
    fi

    if [ "$NOSNAP"x = ""x ] ; then
	if [ $(hostname) = casulana ]; then
	    mkdir -p /tmp/snapshots
	    rsync_to_pettersson /tmp/snapshots/ snapshots/
            publish_on_pettersson snapshots
	else
            ~/bin/weekly-snapshots
	fi
    fi

    catch_live_builds

    for DIR in $RSYNC_TARGET $RSYNC_TARGET_FIRMWARE; do
	date -u > $DIR/trace/cdimage.debian.org
	if [ $(hostname) = casulana ]; then
	    PETOUT=${DIR##${OUT_BASE}/}
	    rsync_to_pettersson  $DIR/trace/ ${PETOUT}/trace/
	    publish_on_pettersson ${PETOUT}/trace
	fi
    done
    rm -f $BUILDLOCK
fi