rocklinux/scripts/Download

#!/bin/bash
#
# --- ROCK-COPYRIGHT-NOTE-BEGIN ---
# 
# This copyright note is auto-generated by ./scripts/Create-CopyPatch.
# Please add additional copyright information _after_ the line containing
# the ROCK-COPYRIGHT-NOTE-END tag. Otherwise it might get removed by
# the ./scripts/Create-CopyPatch script. Do not edit this copyright text!
# 
# ROCK Linux: rock-src/scripts/Download
# ROCK Linux is Copyright (C) 1998 - 2003 Clifford Wolf
# 
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version. A copy of the GNU General Public
# License can be found at Documentation/COPYING.
# 
# Many people helped and are helping developing ROCK Linux. Please
# have a look at http://www.rocklinux.org/ and the Documentation/TEAM
# file for details.
# 
# --- ROCK-COPYRIGHT-NOTE-END ---
#
#   Run this command from the ROCK directory as ./scripts/Download [ options ]
#
#   It enables you to download source files as described in the package 
#   definitions (optionally using a mirroring 'cache' server). 
#
#   This script also allows for checksum display/validation.
#
#   Key to understanding this script is that all file lists get broken
#   down to a single file download using ./scripts/Download filename.

umask 022

if [ "$1" = '--help' ] ; then
	{ echo
	echo "Usage:"
	echo
	echo " ./scripts/Download [options] [ Filename(s) ]"
	echo " ./scripts/Download [options] -pattern Pattern(s)"
	echo " ./scripts/Download [options] -package Package(s)"
	echo " ./scripts/Download [options] -repository Repositories"
	echo " ./scripts/Download [options] { -all | -required }"
	echo
	echo " Where [options] is an alias for:"
	echo "    [ -cfg <config> ] [ -nock ]  [ -alt-dir <AlternativeDirectory> ]"
	echo "    [ -mirror <URL> | -check ]  [ -try-questionable ]  [ -notimeout ]"
	echo "    [ -longtimeout ]  [ -curl-opt <curl-option>[:<curl-option>[:..]] ]"
	echo "    [ -proxy <server>[:<port>] ]"
	echo
	echo " On default, this script auto-detects the best ROCK Linux mirror."
	echo
	echo " Mirrors can also be a local directories in the form of 'file:///<dir>'."
	echo
	echo " ./scripts/Download -mk-cksum Filename(s)"
	echo " ./scripts/Download [ -list | -list-unknown | -list-missing | -list-cksums ]"
	echo
	echo "See '-mirror none' output for help on bypassing the official mirrors."
	echo ; } >&2
	exit 1
fi

# -mk-cksum mode (display ROCK type package checksum): it
# displays the checksum ROCK validates against.
#
# Currently bz2, tbz2, gz, tgz, Z are unpacked
#
if [ "$1" = -mk-cksum ] ; then
    shift
    for x ; do
	echo -n "$x: "
	if [ ! -f "$x" ] ; then
	    echo "No such file."
	elif [ "${x%.bz2}" != "$x" -o "${x%.tbz2}" != "$x" ] ; then
	    bunzip2 < "$x" | cksum | cut -f1 -d' '
	elif [ "${x%.gz}"  != "$x" -o "${x%.tgz}"  != "$x" ] ; then
	    gunzip < "$x" | cksum | cut -f1 -d' '
	elif [ "${x%.Z}" != "$x" ] ; then
	    uncompress < "$x" | cksum | cut -f1 -d' '
	else
	    cksum < "$x" | cut -f1 -d' '
	fi
    done
    exit 1
fi

# Handle options passed on the command line
#
mkdir -p src/ ; config=default
mirror='' ; checkonly=0 ; altdir='' ; loop=1
tryques=0 ; nocheck=0 ; options=''
notimeout=0 ; curl_options='--disable-epsv'
#
while [ $loop -eq 1 ] ; do
	case "$1" in

	    -cfg)
		options="$options -cfg $2"
		config="$2" ; shift ; shift ;;

	    -nock)
		# -nock skips checksum checking (don't use lightly)
		options="$options -nock"
		nocheck=1 ; shift ;;

	    -mirror)
		# -mirror uses a mirror for finding source files
		if [ "$2" = none ]; then
			echo
			echo "The option '-mirror none' is not supported anymore!"
			echo
			echo "You may 'echo none > download/Mirror' if you really"
			echo "want to use the original download resources. However, this"
			echo "is not supported and if such a download fails, this is not"
			echo "a bug in ROCK Linux and doesn't neccessarily needs fixing."
			echo
			exit 1;
		else
			echo "$2" > download/Mirror
			options="$options -mirror $2"
			mirror="$2"
		fi
		shift ; shift ;;

	    -check)
		# -check just validates the file using the checksum
		options="$options -check"
		checkonly=1 ; shift ;;

	    -notimeout)
		# don't add timeout curl options
		options="$options -notimeout"
		notimeout=2 ; shift ;;

	    -longtimeout)
		# don't add timeout curl options
		options="$options -longtimeout"
		notimeout=1 ; shift ;;

	    -curl-opt)
		# additional curl options
		options="$options -curl-opt $2"
		curl_options="$curl_options `echo $2 | tr : ' '`"
		shift ; shift ;;

	    -proxy)
		# proxy option for curl
		echo -n "$2" > download/Proxy
		options="$options -proxy $2"
		shift ; shift ;;

	    -alt-dir)
		# check for an alternative directory where to search for
		# package source tarballs
		options="$options -alt-dir $2"
		altdir=$2 ; shift ; shift ;;

	    -try-questionable)
		# also try to download questionable URLs
		options="$options -try-questionable"
		tryques=1 ; shift ;;

	    *)
		loop=0 ;;
	esac
done

if [ $notimeout -eq 0 ] ; then
	curl_options="$curl_options -y 10 -Y 10 --connect-timeout 60"
fi
if [ $notimeout -eq 1 ] ; then
	curl_options="$curl_options -y 60 -Y 1  --connect-timeout 300"
fi

if [ -f download/Proxy ]; then
	proxy="`cat download/Proxy`"
	if [ "$proxy" ]; then
		echo "INFO: Setting proxy to $proxy."
		curl_options="$curl_options --proxy $proxy"
	else
		echo "INFO: Empty proxy definition... removing download/Proxy."
		rm download/Proxy
	fi
fi

# cksum_chk filename cksum origfile
#
# This function verifies the checksum. If it fails it renames the file
# to file.chksum-err and returns failure.
#
# It seams like the [ ] command has problems with comparing high numbers.
# That's why I'm using a text comparison here.
#
# Not doing anything if checksum is '0' or a text of 'X'.
#
cksum_chk() {
	y="`echo $2 | sed 's,^0*,,;'`"
	[ $nocheck = 1 -o -z "$y" -o -z "${2//X/}" ] && return 0
	x="`cksum "$1" | cut -f1 -d' ' | sed 's,^0*,,;'`"
	if [ "$x" != "$y" ] ; then
	    # Add .cksum-err extension to filename:
		echo "Cksum ERROR: $3.cksum-err ($x)"
		mv "$3" "$3.cksum-err" ; return 1
	fi
	return 0
}

# Autodetect best Mirror and safe url in $mirror
#
detect_mirror() {
    if [ -f download/Mirror ] ; then
	mirror="`cat download/Mirror`"
	if [ -z "$mirror" -o "$mirror" = "none" ] ; then
		echo "INFO: Found download/Mirror: none" \
			"(use the original download locations)"
	else
		echo "INFO: Found cached mirror URL in download/Mirror:"
		echo "INFO: $mirror"
	fi
	echo "INFO: To force a new mirror auto-detection, remove download/Mirror."
    else
	echo "INFO: Auto-detecting best mirror ..."
	eval "$(egrep '^(rockver)=' scripts/parse-config)"

	echo "INFO: Downloading mirror-list from www.rocklinux.net."
	curl -s -S $curl_options -o src/Download-Mirror-List \
		"http://www.rocklinux.net/mirrors.cgi?$rockver"

	bestval=0 ; result='No Mirror Found!'
	while read mirror_name ; do
	  if [ "${mirror_name#=}" != "$mirror_name" ] ; then
		mirror_name="${mirror_name#= }"
		mirror_name="${mirror_name% =}"
		read mirror_url
		echo -n "INFO: Testing <$mirror_name> ..."
		val="$(curl -s $curl_options -m 20 "${mirror_url%/}/DOWNTEST" \
			-w "ok %{speed_download}" -o /dev/null)"
		if [ "$val" = "${val#ok }" -o "$val" = "ok 0.000" ] ; then
			echo " error"
		else
			xval=`echo ${val#ok } | tr -d .` ; echo " $val"
			if [ "$xval" -gt "$bestval" ] ; then
				bestval=$xval ; mirror="${mirror_url%/}"
				result="Using mirror <$mirror>."
			fi
		fi
	  fi
	done < src/Download-Mirror-List
	echo $mirror > download/Mirror
	echo "INFO: $result"
    fi
}

# download_file local-filename download-location cksum
#
# This function executes the actual download using curl.
#
download_file() {

	# Init
	#
	gzfile="$1" ; location="$2" ; cksum="$3" 
	# Make src directory for creating tar balls
	mkdir -p src/
	# Tar ball file name:
	bzfile="`echo "$gzfile" | sed 's,\.\(t\?\)gz$,.\1bz2,'`"
	# Lock file name:
	lkfile="src/down.lockfile.`echo $bzfile | tr / -`"

	# Check if it's already there
	#
	[ -s "$bzfile" -a $checkonly != 1 ] && return 0

	# Make locking
	#
	if [ -s "$lkfile" ]; then
		echo "Found $lkfile -> skip download."
		return 0
	fi
	trap 'rm -f "$lkfile"' INT
	echo $$ > "$lkfile"

	# Check if we only like to test the cksum(s)
	#
	if [ $checkonly = 1 ] ; then
		gzfile="$bzfile"
		if [ ! -f "$bzfile" ] ; then
			echo "File missing: $bzfile"
			rm -f "$lkfile" ; trap INT ; return 1
		fi
		if [ -z "${cksum##X*}" ] ; then
			echo "No checksum (ignore): $bzfile"
			rm -f "$lkfile" ; trap INT ; return 1
		fi
		if [ "$cksum" -eq 0 ] ; then
			echo "No checksum (missing): $bzfile"
			rm -f "$lkfile" ; trap INT ; return 1
		fi

	elif [ -s "$gzfile" ] ; then

		echo ; echo "Already downloaded $gzfile ..."

	else

		echo ; echo "Downloading $gzfile ..."

		# Existing *.cksum-err
		#
		if [ -s "$gzfile.cksum-err" ] ; then
			# cksum-err file alread exists:
			echo "ERROR: Found $gzfile.cksum-err."
			echo "ERROR: That means that we downloaded the" \
			     "file already and it had an"
			echo "ERROR: incorrect checksum. Remove the" \
			     "*.cksum-err file to force a"
			echo "ERROR: new download of that file."
			rm -f "$lkfile" ; trap INT ; return 1
		fi

		# Existing *.extck-err
		#
		if [ -s "$gzfile.extck-err" ] ; then
			# extck-err file alread exists:
			echo "ERROR: Found $gzfile.extck-err."
			echo "ERROR: That means that we downloaded the" \
			     "file already and it's content"
			echo "ERROR: did not match it's filename extension." \
			     "Remove the *.extck-err file"
			echo "ERROR: to force a new download of that file."
			rm -f "$lkfile" ; trap INT ; return 1
		fi

		# Questionable URL
		#
		if [ "$location" != "${location#\?}" ] ; then
			if [ "$tryques" = 0 ] ; then
				echo "ERROR: URL is marked as questionable." \
					"Not downloading this file."
				rm -f "$lkfile" ; trap INT ; return 1
			else
				echo "WARNING: URL is marked as questionable." \
					"Downloading it anyways."
				location="${location#\?}"
			fi
		fi

		# Make directory (if required)
		#
		if [ ! -d `dirname "$bzfile"` ] ; then
			mkdir -p `dirname "$bzfile"`
		fi
		
		# Alternative Directory
		#
		if [ "$altdir" ] ; then
		    altfile=$(find $altdir/ -name `basename $bzfile` | head -1)
		else
		    altfile=""
		fi

		if [ "$altfile" ] ; then

			echo "Found `basename $bzfile` as $altfile."
			cp -lv $altfile $bzfile ; gzfile="$bzfile"

		else

		    # Mirroring
		    #
		    [ -z "$mirror" ] && detect_mirror
		    if [ "$mirror" -a "$mirror" != "none" ] ; then
			location="!$mirror/${bzfile#download/}"
			gzfile="$bzfile"
		    fi
		
		    # Create URL
		    #
		    if [ "${location#!}" != "$location" ] ; then
			url="`echo "$location" | sed 's,!,,'`"
		    else
			url="`echo "$location" | \
			     sed 's,/[^/]*$,,'`/`echo $gzfile | sed 's,.*/,,'`"
		    fi

		    # Check for existing Error Log
		    #
		    if test -s src/Download-Errors &&
		       grep -q " $url\$" src/Download-Errors ; then
			    echo "ERROR: According to src/Download-Errors" \
			         "we had already an error for that URL."
			    echo "ERROR: So I'm not trying to download" \
			         "it again (remove src/Download-Errors"
			    echo "ERROR: if you want to force a retry)."
			    rm -f "$lkfile" ; trap INT ; return 1
		    fi

		    # Download
		    #
		    if [[ $url = cvs://* ]] ; then

			url="`dirname $url`"
			mode="`echo $url | sed -e s,^cvs://,, -e 's,:.*,,'`"
			
			if [ "${url##*\!*}" ] ; then dat=""
			else dat="-D ${url##*\!}" ; dat="${dat//_/ }" ; fi

			loc="`echo $url | \
			  sed 's,^cvs://[a-z,A-Z]*:,,; s,::.*$,,'`"
			module="`echo $url | sed 's/^.*:://;   s,!.*$,,'`"

			cvsdir="src/down.cvsdir.`echo $bzfile | tr / -`"
			saved_pwd=$PWD ; mkdir -p $cvsdir ; cd $cvsdir

			echo CVS $mode $loc $dat $module
			{ [ $mode = ssh ] && export CVS_RSH=ssh
			  [ $mode = pserver ] && loc=":pserver:$loc"
			  # for ssh we need some way to quitely accept
			  # the key ...
			  echo cvs -z9 -Q -d $loc checkout $dat -P $module
			  if ! cvs -z9 -Q -d $loc checkout $dat -P $module
			  then touch .cvs_error ; fi
			} &> .cvs_output &

			while fuser .cvs_output &> /dev/null ; do
				echo -ne `nice du -sh 2> /dev/null | cut -f1` 'downloaded from' \
					'CVS archive so far...\r'
				sleep 3
			done
			echo `du -sh 2> /dev/null | cut -f1` 'downloaded from' \
				'CVS archive (download finished).'

			if [ ! -f .cvs_error ] ; then
				cd `dirname $module`
				dir="`echo "$bzfile" | sed s/\.tar\.bz2$//`"
				dir="`basename $dir`"

				mv `basename $module` $dir

				tar --owner root --group root \
				    --use-compress-program=bzip2 \
				    -cf $dir.tar.bz2 $dir
				mv $dir.tar.bz2 $saved_pwd/$bzfile

				cd $saved_pwd ; rm -rf $cvsdir
			else
				cat .cvs_output
				cd $saved_pwd ; rm -rf $cvsdir
				echo ERROR: CVS $dat $loc $module \
				            returned an error.
				echo "0 $gzfile $url" >> src/Download-Errors
			fi
		    else
			if [ -s "$gzfile.incomplete" ] ; then
			    echo "INFO: Trying to resume previous download .."
			    resume="-C -"
			else
			    resume=""
			fi

			curl -w '\rFinished downloading %{size_download} bytes in %{time_total} seconds (%{speed_download} bytes/sec). \n' -f --progress-bar $resume $curl_options "$url" -o "$gzfile.incomplete"
			curlret="$?"

			if [ "$resume" ] && \
			   [ $curlret -eq 33 -o $curlret -eq 36 ] ; then
			    echo "INFO: Resuming download not possible. ->" \
			         "Overwriting old file."
			    rm -f "$gzfile.incomplete"
			    curl -w '\rFinished downloading %{size_download} bytes in %{time_total} seconds (%{speed_download} bytes/sec). \n' -f --progress-bar $curl_options "$url" -o "$gzfile.incomplete"
			    curlret="$?"
			fi

			if [ $curlret -ne 0 ] ; then
			    rm -f "$lkfile" ; trap INT
			    case "$curlret" in
			      18)
			      echo "WARNING: Got only some of the" \
				   "file. A re-run of $0"
			      echo "WARNING: is required to complete" \
				   "the download." ;;
			      130)
			      echo -e '\rWARNING: CURL got a SIGINT' \
				   "(someone pressed Ctrl-C). A re-run of"
			      echo "WARNING: $0 is required to complete" \
				   "the download." ; sleep 1 ;;
			      *)
			      echo "$curlret $gzfile $url" \
						>> src/Download-Errors
			      echo -e '\rERROR: CURL Returned Error' \
				   "$curlret. Please read" \
				   "the curl manpage." ;;
			    esac
			    return 1
			elif [ ! -s "$gzfile.incomplete" ] ; then
			    echo "0 $gzfile $url" >> src/Download-Errors
			    echo "ERROR: CURL returned success but" \
			         "we have no data!"
			    curlret=1
			else
			    case "$gzfile" in
			      *.gz|*.tgz)
				  typeexpr="gzip compressed data" ;;
			      *.bz2|*.tbz2)
				  typeexpr="bzip2 compressed data" ;;
			      *.Z)
				  typeexpr="compress'd data" ;;
			      *.zip|*.jar)
				  typeexpr="Zip archive data" ;;
			      *.tar)
				  typeexpr="tar archive" ;;
			      *)
				  echo "WARNING: Unkown file extension: $gzfile"
				  typeexpr="." ;;
			    esac
			    if file "$gzfile.incomplete" | grep -v "$typeexpr"
			    then
				echo "ERROR: File type does not match" \
				     "filename ($typeexpr)!"
				mv "$gzfile.incomplete" "$gzfile.extck-err"
			    else
				mv "$gzfile.incomplete" "$gzfile"
			    fi
		        fi
		    fi
		fi

		if [ ! -s "$gzfile" ]; then
			rm -f "$lkfile" ; trap INT ; return 1
		fi
	fi

	# Convert a .gz to .bz2 and test checksum
	#
	if [ "$gzfile" != "$bzfile" ] ; then
		echo "gzip->bzip2 + cksum-test: $gzfile"
		gunzip < "$gzfile" > src/down.$$.dat
		if cksum_chk src/down.$$.dat $cksum "$gzfile" ; then
			bzip2 < src/down.$$.dat > "$bzfile" ; rm -f "$gzfile"
		fi
		rm -f src/down.$$.dat

	# Execute a cksum test on a bzip2 file
	#
	elif [ "${gzfile%.bz2}"  != "$gzfile" -o \
	       "${gzfile%.tbz2}" != "$gzfile" ]
	then
		echo "cksum-test (bzip2): $bzfile"
		if [ $nocheck = 0 ] ; then
			bunzip2 < "$bzfile" > src/down.$$.dat
			cksum_chk src/down.$$.dat $cksum "$bzfile"
		fi
		rm -f src/down.$$.dat

	# Execute a cksum test on a raw data file
	#
	else
		echo "cksum-test (raw): $gzfile"
		cksum_chk "$gzfile" $cksum "$gzfile"
	fi

	# Free Lock and finish
	#
	rm -f "$lkfile" ; trap INT ; return 0
}

# handle_file filename
#
# This function fetches the checksum and download information 
# from the desc files and calls the download_file function.
#
handle_file() {

	# Handle pkg file (package/.../*.desc)
	#
	tree="`echo "$1" | cut -f2 -d/`"
	pkg="` echo "$1" | cut -f3 -d/`"
	file="`echo "$1" | cut -f4 -d/`"
	if grep -q "^\[D\].* $file " package/$tree/$pkg/$pkg.desc 2> /dev/null
	then
		grep "^\[D\].* $file " package/$tree/$pkg/$pkg.desc |
		while read tag cksum file url ; do
		    download_file "$1" "$url" "$cksum"
		done
		return 0
	fi

	# Handle target file (target/.../download.txt)
	#
	target="`echo "$1" | cut -f2 -d/`"
	file="`echo  "$1" | cut -f3- -d/`"
	if grep -q " $file " target/$target/download.txt 2> /dev/null
	then
		grep " $file " target/$target/download.txt |
		while read cksum file url ; do
		    download_file "$1" "$url" "$cksum"
		done
		return 0
	fi

	# Handle misc file (scripts/miscdown.txt)
	#
	if [ "${1#download/misc/}" != "$1" ] ; then
	    file="`echo "$1" | cut -f3- -d/`"
	    if grep -q " $file " scripts/miscdown.txt
	    then
		grep " $file " scripts/miscdown.txt |
		while read cksum file url ; do
		    download_file "$1" "$url" "$cksum"
		done
		return 0
	    fi
	fi

	# Can't handle file / file not found
	#
	return 1
}

list_cksums() {
	trap '' INT
	{ grep -H '^\[D\] ' package/*/*/*.desc | tr '\t' ' ' | tr -s ' ' |
		sed -e 's,^package/,download/,;'
	  grep -H '^[X0-9]' target/*/download.txt | tr '\t' ' ' | tr -s ' ' |
		sed -e 's,^target/,download/,; s,:,:[D] ,;'
	  grep -H '^[X0-9]' scripts/miscdown.txt | tr '\t' ' ' | tr -s ' ' |
		sed 's,^scripts/,download/misc/,; s,:,:[D] ,;'
	} | sed 's,^\(.*/\)[^/:]*:[^ ]* \([X0-9]*\) ,\2 \1,;' | cut -f1,2 -d' '
	trap INT
}

list() {
	trap '' INT
	list_cksums | cut -f2- -d' '
	trap INT
}

list_unknown() {
	trap '' INT
	mkdir -p src/ ; list | sed 's,\.\(t\?\)gz$,.\1bz2,' > src/down.$$.lst
	ls download/{INDEX,README,DOWNTEST,LAST-UPDATE} \
						>> src/down.$$.lst 2> /dev/null
	find download/* -type f -o -type l | \
	while read fn ; do
		grep -qx "$fn" src/down.$$.lst || echo "Unknown file: $fn"
	done
	rm -f src/down.$$.lst
	trap INT
}

list_missing() {
	trap '' INT
	list | sed 's,\.\(t\?\)gz$,.\1bz2,' | \
	while read fn ; do
		[ -f "$fn" ] || echo "$fn"
	done
	trap INT
}

pattern() {
	for pattern ; do
		echo "Processing pattern $pattern ..."
		list | while read fn ; do
			if [ "${fn##$pattern}" = '' ] ; then
				echo "Matched: $fn"
				$0 $options "$fn"
			fi
		done
	done
}

package() {
	for package ; do
		pattern "download/*/$package/*"
	done
}

repository() {
	for repository ; do
		pattern "download/$repository/*"
	done
}

required() {
    while read on a b tree pkg c ; do
	if [ "$on" = "X" ] ; then
	    grep -H '^\[D\] ' package/$tree/$pkg/$pkg.desc > src/down.$$.lst
	    while read tag cksum file url ; do
		download_file "download/$tree/$pkg/$file" "$url" "$cksum"
	    done < src/down.$$.lst ; rm -f src/down.$$.lst
	fi
    done < config/$config/packages
    target=`grep '^export ROCKCFG_TARGET=' config/$config/config | \
						cut -f2 -d= | tr -d "'"`
    if [ -f target/$target/download.txt ] ; then
	while read cksum file url ; do
		download_file "download/$target/$file" "$url" "$cksum"
	done < target/$target/download.txt
    fi
}

single_files() {
    for file_name ; do
	if ! handle_file "$file_name" ; then
		file_name="`echo "$file_name" | sed 's,\.\(t\?\)bz2$,.\1gz,'`"
		if ! handle_file "$file_name" ; then
			echo "ERROR: Unknown file: $file_name."
		fi
	fi
    done
}

all() {
	if [ $checkonly = 1 ] ; then list
	else list_missing ; fi > src/down.$$.lst
	while read fn ; do single_files $fn ; done < src/down.$$.lst
	rm -f src/down.$$.lst
}

case "$1" in
	-list)		list ;;
	-list-unknown)	list_unknown ;;
	-list-missing)	list_missing ;;
	-list-cksums)	list_cksums ;;

	-pattern)	shift ; pattern "$@" ;;
	-package)	shift ; package "$@" ;;
	-repository)	shift ; repository "$@" ;;
	-required)	required ;;
	-all)		all ;;

	-*|"")		exec $0 --help ;;
	*)		single_files "$@" ;;
esac

exit 0