build/download: another new helper

Also fetch logs from a remote build server, but skip all uncompressed
directories because it slows down the download.

Next are two minor modications for nightly logging that reduce the
size of the log history and should compress the logs right after
the build is done.

Makefile

@@ -23,10 +23,10 @@
 # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 # SUCH DAMAGE.
 
-STEPS=		arm base boot chroot clean core distfiles dvd info \
-		kernel nano plugins ports prefetch print rebase \
-		release rename serial sign skim test update upload \
-		verify vga vm xtools
+STEPS=		arm base boot chroot clean core distfiles download \
+		dvd info kernel nano plugins ports prefetch print \
+		rebase release rename serial sign skim test update \
+		upload verify vga vm xtools
 SCRIPTS=	batch nightly refresh pkg_fingerprint pkg_sign
 .PHONY:		${STEPS}
 

README.md

@@ -397,6 +397,13 @@ To push sets and images to a remote location use the upload target:
 
     # make upload-<user@sshtarget>,<set_or_image>[,...]
 
+To pull sets and images from a remote location use the download target:
+
+    # make download-<user@sshtarget>,<set_or_image_or_logs>[,...]
+
+Logs can be downloaded as well for local inspection.  Note that download
+like prefetch will purge all locally existing targets.
+
 Last but not least, a refresh of OPNsense core and plugins on package
 sets is invoked using:
 

build/download.sh

@@ -0,0 +1,69 @@
+#!/bin/sh
+
+# Copyright (c) 2018 Franco Fichtner <franco@opnsense.org>
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+
+set -e
+
+SELF=download
+
+. ./common.sh
+
+if [ -z "${1}" ]; then
+	echo ">> Missing download target"
+	exit 1
+fi
+
+SSHTARGET=${1}
+
+# remove first argument
+shift
+
+download()
+{
+	echo ">>> Downloading ${1}..."
+	scp ${SSHTARGET}:"${2}/${3}" ${2}
+}
+
+for ARG in ${@}; do
+	case ${ARG} in
+	arm|dvd|nano|serial|vga|vm)
+		sh ./clean.sh ${ARG}
+		download ${ARG} ${IMAGESDIR} "${ARG}-*-${PRODUCT_FLAVOUR}-*"
+		;;
+	base|kernel)
+		sh ./clean.sh ${ARG}
+		download ${ARG} ${SETSDIR} "${ARG}-*"
+		;;
+	logs)
+		sh ./clean.sh ${ARG}
+		download ${ARG} ${LOGSDIR} "[0-9]*.tgz"
+		;;
+	packages|release)
+		sh ./clean.sh ${ARG}
+		download ${ARG} ${SETSDIR} "${ARG}-*-${PRODUCT_FLAVOUR}-*"
+		;;
+	esac
+done