Juergen Sawinski:

scripts/Build-Pkg

@@ -246,8 +246,7 @@ if [ "$ROCKCFG_RETRY_BROKEN" = 0 -a $norebuild = 1 -a \
 	exit 1
 fi
 
-confdir="" ; archdir=""
-
+confdir=""
 if [ -z "$pkgdir" ] ; then
   for x in package/*/$pkg/$pkg.desc ; do
     if [ -f "$x" ] ; then
@@ -259,14 +258,12 @@ if [ -z "$pkgdir" ] ; then
 	fi
 	x=${x#package/}; x=${x%%/*}
 	confdir="$base/package/$x/$pkg"
-	archdir="$base/download/$x/$pkg"
 	repository=$x
     fi
   done
 else
   if [ -f "$pkgdir/$pkg.desc" ] ; then
 	confdir="$pkgdir"
-	archdir="$pkgdir"
 	repository=extern
   fi
 fi
@@ -572,6 +569,13 @@ elif [ "$ROCKCFG_FLIST" = "find" ] ; then
 	sleep 2
 fi
 
+echo_status "Creating source archive directory."
+archdir="$builddir/archdir"; mkdir -p "$archdir"
+while read l_cksum l_srctar l_url l_flags ; do
+    [ -z "$l_cksum" ] && continue 
+    ln -s `source_file l_cksum $l_srctar l_url $l_flags` $archdir/
+done < <( echo "$desc_D" | column_clean )
+
 hook_eval prepare
 
 # define new abort function for errors while building

scripts/Create-CkSumPatch

@@ -52,10 +52,16 @@ while read cksum gzfile ; do
    if [ -f "$bzfile" ]
    then
 	checkthis=1
-	y="`echo $bzfile | cut -f$cmpfield -d/`"
+
+	xfile="`basename $gzfile`"
+	descfile="`grep $xfile package/*/*/*.desc target/*/download.txt scripts/miscdown.txt  | sed 's,:.*,,'`"
+	rep="`echo $descfile | cut -f2 -d/`" 
+	pkg="`echo $descfile | cut -f3 -d/`" 
+
 	for x ; do
 		[ $checkthis = 1 ] && checkthis=0
-		[ "$y" = $x ] && checkthis=2
+		[ "$pkg" = $x ] && checkthis=2
+		[ "$descfile" = $x ] && checkthis=2
 	done
 	if [ $checkthis = 0 ] ; then
 		: echo "Skip: $bzfile" ; continue
@@ -70,13 +76,7 @@ while read cksum gzfile ; do
 	fi
 	echo $cksum >&2
 
-	rep="`echo $bzfile | cut -f2 -d/`"
-
 	if [ -d package/$rep ] ; then
-		xfile="`echo $gzfile | cut -f4- -d/`"
-		pkgdir="package/`echo $gzfile | cut -f2,3 -d/`"
-		descfile="$pkgdir/`echo $gzfile | cut -f3 -d/`.desc"
-
 		if [ -f $descfile ] ; then
 			sedscript="s,\[D\]  *00*  *$xfile,[D] $cksum $xfile,"
 			sed "$sedscript" < $descfile > /tmp/$$
@@ -93,13 +93,6 @@ while read cksum gzfile ; do
 			echo "!!! (My guess was $descfile)" >&2
 		fi
 	elif [ -d target/$rep -o "$rep" = misc ] ; then
-		if [ -d target/$rep ] ; then
-			descfile="target/$rep/download.txt"
-		else
-			descfile="scripts/miscdown.txt"
-		fi
-		xfile="`echo $gzfile | cut -f3- -d/`"
-
 		if [ -f $descfile ] ; then
 			sedscript="s,^00*  *$xfile,$cksum $xfile,"
 			sed "$sedscript" < $descfile > /tmp/$$

scripts/Download

@@ -28,19 +28,17 @@
 #   definitions (optionally using a mirroring 'cache' server). 
 #
 #   This script also allows for checksum display/validation.
-#
-#   Key to understanding this script is that all file lists get broken
-#   down to a single file download using ./scripts/Download filename.
 
 umask 022
 
+. scripts/functions
+
 if [ "$1" = '--help' ] ; then
 	{ echo
 	echo "Usage:"
 	echo
-	echo " ./scripts/Download [options] [ Filename(s) ]"
-	echo " ./scripts/Download [options] -pattern Pattern(s)"
-	echo " ./scripts/Download [options] -package Package(s)"
+	echo " ./scripts/Download [options] [ Package(s) ]"
+	echo " ./scripts/Download [options] [ Desc file(s) ]"
 	echo " ./scripts/Download [options] -repository Repositories"
 	echo " ./scripts/Download [options] { -all | -required }"
 	echo
@@ -49,6 +47,7 @@ if [ "$1" = '--help' ] ; then
 	echo "    [ -mirror <URL> | -check ]  [ -try-questionable ]  [ -notimeout ]"
 	echo "    [ -longtimeout ]  [ -curl-opt <curl-option>[:<curl-option>[:..]] ]"
 	echo "    [ -proxy <server>[:<port>] ] [ -proxy-auth <username>[:<password>] ]"
+	echo "    [ -copy ] [ -move ]"
 	echo
 	echo " On default, this script auto-detects the best ROCK Linux mirror."
 	echo
@@ -93,6 +92,8 @@ this_is_the_2nd_run=0
 mirror='' ; checkonly=0 ; altdir='' ; loop=1
 tryques=0 ; nocheck=0 ; options='-this_is_the_2nd_run '
 notimeout=0 ; curl_options='--disable-epsv --location'
+altcopy=link
+
 #
 while [ $loop -eq 1 ] ; do
 	case "$1" in
@@ -176,6 +177,9 @@ while [ $loop -eq 1 ] ; do
 		# also try to download questionable URLs
 		options="$options -try-questionable"
 		tryques=1 ; shift ;;
+	
+	    -move) altcopy=move ; shift ;;
+	    -copy) altcopy=copy ; shift ;;
 
 	    *)
 		loop=0 ;;
@@ -260,7 +264,7 @@ detect_mirror() {
     fi
 }
 
-# download_file local-filename download-location cksum
+# download_file local-filename download-location cksum repo pkg
 #
 # This function decides if download directly or from a mirror,
 # validates checksum, etc.
@@ -270,7 +274,7 @@ download_file() {
 
 	# Init
 	#
-	gzfile="$1" ; location="$2" ; cksum="$3" 
+	local gzfile="$1" location="$2" cksum="$3" repo="$4" pkg="$5"
 	# Make src directory for creating tar balls
 	mkdir -p src/
 	# Tarball file name: (if you change this one - also adapt Create-ISO)
@@ -370,10 +374,25 @@ download_file() {
 		    altfile=""
 		fi
 
+		#FIXME: compatibility, can be removed sooner or later...
+		# Check old download dir layout
+		if [ -z "$altfile" ]; then
+			if [ -f "download/$repo${pkg:+/}$pkg/`basename $bzfile`" ]; then
+				altfile="download/$repo${pkg:+/}$pkg/`basename $bzfile`"
+			fi
+		fi
+
 		if [ "$altfile" ] ; then
 
 			echo "Found `basename $bzfile` as $altfile."
-			cp -lv $altfile $bzfile ; gzfile="$bzfile"
+			if [ "$altcopy" = 'link' ]; then
+				cp -lv $altfile $bzfile
+			elif [ "$altcopy" = 'copy' ]; then
+				cp -v $altfile $bzfile
+			elif [ "$altcopy" = 'move' ]; then
+				mv -v $altfile $bzfile
+			fi
+			gzfile="$bzfile"
 
 		else
 
@@ -383,10 +402,15 @@ download_file() {
 
 		    if [ "$mirror" -a "$mirror" != "none" ] ; then
 			# try to use mirror
+
 			if ! download_file_now "!$mirror/${bzfile#download/}" $bzfile $bzfile; then
-				# oops... so try direct (original URL)
-				echo "INFO: download from mirror failed, trying original URL."
-				download_file_now $location $gzfile $bzfile
+				#FIXME: compatibility, can be removed sooner or later...
+				# Try old download format for remote file
+				if ! download_file_now "!$mirror/$repo${pkg:+/}$pkg/`basename $bzfile`" $bzfile $bzfile; then
+					# oops... so try direct (original URL)
+					echo "INFO: download from mirror failed, trying original URL."
+					download_file_now $location $gzfile $bzfile
+				fi
 			else
 				gzfile="$bzfile"
 			fi
@@ -456,7 +480,8 @@ download_file_now() {
 	if test -s src/Download-Errors &&
  	   grep -q " $url\$" src/Download-Errors ; then
 		echo "ERROR: According to src/Download-Errors" \
-		     "we had already an error for that URL."
+		     "we had already an error for the URL"
+		echo "ERROR:    $url"
 		echo "ERROR: So I'm not trying to download" \
 		     "it again (remove src/Download-Errors"
 		echo "ERROR: if you want to force a retry)."
@@ -601,69 +626,25 @@ download_file_now() {
 	fi
 }
 
+list_dtags() {
+	{ 
+		grep -H '^\[D\] ' package/*/*/*.desc
+		grep -H '^[X0-9]' target/*/download.txt | sed 's,:,:[D] ,'
 
-# handle_file filename
-#
-# This function fetches the checksum and download information 
-# from the desc files and calls the download_file function.
-#
-handle_file() {
-
-	# Handle pkg file (package/.../*.desc)
-	#
-	tree="`echo "$1" | cut -f2 -d/`"
-	pkg="` echo "$1" | cut -f3 -d/`"
-	file="`echo "$1" | cut -f4 -d/`"
-	if grep -q "^\[D\].* $file " package/$tree/$pkg/$pkg.desc 2> /dev/null
-	then
-		grep "^\[D\].* $file " package/$tree/$pkg/$pkg.desc |
-		while read tag cksum file url ; do
-		    download_file "$1" "$url" "$cksum"
-		done
-		return 0
-	fi
-
-	# Handle target file (target/.../download.txt)
-	#
-	target="`echo "$1" | cut -f2 -d/`"
-	file="`echo  "$1" | cut -f3- -d/`"
-	if grep -q " $file " target/$target/download.txt 2> /dev/null
-	then
-		grep " $file " target/$target/download.txt |
-		while read cksum file url ; do
-		    download_file "$1" "$url" "$cksum"
-		done
-		return 0
-	fi
-
-	# Handle misc file (scripts/miscdown.txt)
-	#
-	if [ "${1#download/misc/}" != "$1" ] ; then
-	    file="`echo "$1" | cut -f3- -d/`"
-	    if grep -q " $file " scripts/miscdown.txt
-	    then
-		grep " $file " scripts/miscdown.txt |
-		while read cksum file url ; do
-		    download_file "$1" "$url" "$cksum"
-		done
-		return 0
-	    fi
-	fi
-
-	# Can't handle file / file not found
-	#
-	return 1
+		#FIXME the following have relative paths in the archive-file column
+		# is it needed anyway?
+		grep -H '^[X0-9]' scripts/miscdown.txt | sed 's,:,:[D] ,'
+	} | column_clean
 }
 
 list_cksums() {
 	trap '' INT
-	{ grep -H '^\[D\] ' package/*/*/*.desc | tr '\t' ' ' | tr -s ' ' |
-		sed -e 's,^package/,download/,;'
-	  grep -H '^[X0-9]' target/*/download.txt | tr '\t' ' ' | tr -s ' ' |
-		sed -e 's,^target/,download/,; s,:,:[D] ,;'
-	  grep -H '^[X0-9]' scripts/miscdown.txt | tr '\t' ' ' | tr -s ' ' |
-		sed 's,^scripts/,download/misc/,; s,:,:[D] ,;'
-	} | sed 's,^\(.*/\)[^/:]*:[^ ]* \([X0-9]*\) ,\2 \1,;' | cut -f1,2 -d' '
+
+	list_dtags | sed \
+			-e "s,^$Dre[ ].*\($NODISTre\).*$,\3 download/nodist/\5/\4,;" \
+			-e "s,^$Dre$,\3 download/mirror/\5/\4,;" \
+		| sed 's,^\(.*/\)[^/:]*:[^ ]* \([X0-9]*\) ,\2 \1,;' | cut -f1,2 -d' '
+
 	trap INT
 }
 
@@ -696,27 +677,9 @@ list_missing() {
 	trap INT
 }
 
-pattern() {
-	for pattern ; do
-		echo "Processing pattern $pattern ..."
-		list | while read fn ; do
-			if [ "${fn##$pattern}" = '' ] ; then
-				echo "Matched: $fn"
-				$0 $options "$fn"
-			fi
-		done
-	done
-}
-
-package() {
-	for package ; do
-		pattern "download/*/$package/*"
-	done
-}
-
 repository() {
 	for repository ; do
-		pattern "download/$repository/*"
+		packages `echo package/$repository/*/*.desc`
 	done
 }
 
@@ -729,41 +692,96 @@ required() {
         exit 1
     fi
 
-    while read on a b tree pkg c ; do
+    while read on a b repo pkg c ; do
 	if [ "$on" = "X" ] ; then
-	    grep -H '^\[D\] ' package/$tree/$pkg/$pkg.desc > src/down.$$.lst
-	    while read tag cksum file url ; do
-		download_file "download/$tree/$pkg/$file" "$url" "$cksum"
+	    grep -H '^\[D\] ' package/$repo/$pkg/$pkg.desc > src/down.$$.lst
+	    while read tag cksum file url flags ; do
+		download_file "`source_file cksum $file url $flags`" "$url" "$cksum" "$repo" "$pkg"
 	    done < src/down.$$.lst ; rm -f src/down.$$.lst
 	fi
     done < config/$config/packages
     target=`grep '^export ROCKCFG_TARGET=' config/$config/config | \
 						cut -f2 -d= | tr -d "'"`
     if [ -f target/$target/download.txt ] ; then
-	while read cksum file url ; do
-		download_file "download/$target/$file" "$url" "$cksum"
+	while read cksum file url flags ; do
+	    download_file "`source_file cksum $file url $flags`" "$url" "$cksum" "$target"
 	done < target/$target/download.txt
     fi
+
+    #FIXME what's about scripts/miscdown.txt?
 }
 
-single_files() {
-    for file_name ; do
-	file_name_gz="`echo "$file_name" | sed 's,\.\(t\?\)bz2$,.\1gz,'`"
-	file_name_Z="`echo "$file_name" | sed 's,\.\(t\?\)bz2$,.\1Z,'`"
-	if ! handle_file "$file_name"    && \
-	   ! handle_file "$file_name_gz" && \
-	   ! handle_file "$file_name_Z"
-	then
-		echo "ERROR: Unknown file: $file_name."
-	fi
+all() {
+    local each repo pkg
+
+    for each in package/*/*/*.desc; do
+	pkg="`echo $descfile | cut -f3 -d/`"
+	repo="`echo $descfile | cut -f2 -d/`"
+
+	while read tag cksum file url flags ; do
+	    download_file "`source_file cksum $file url $flags`" "$url" "$cksum" "$repo" "$pkg"
+	done < <(grep -H '^\[D\] ' package/$repo/$pkg/$pkg.desc)
+    done
+    
+    for each in target/*/download.txt; do
+	target="`echo $descfile | cut -f2 -d/`"
+
+	while read cksum file url flags ; do
+	    download_file "`source_file cksum $file url $flags`" "$url" "$cksum" "$target"
+	done < <(cat $each)
+    done
+
+    #FIXME miscdown.txt contains relative paths in 2nd column
+    for each in scripts/miscdown.txt; do
+	while read cksum file url flags ; do
+	    file="`basename $file`"
+	    download_file "`source_file cksum $file url $flags`" "$url" "$cksum" "$target"
+	done < <(cat $each)
     done
 }
 
-all() {
-	if [ $checkonly = 1 ] ; then list
-	else list_missing ; fi > src/down.$$.lst
-	while read fn ; do single_files $fn ; done < src/down.$$.lst
-	rm -f src/down.$$.lst
+packages() {
+	local descfile
+	for arg; do
+		case "$arg" in
+		target/*)
+			if [ ! -f $arg ]; then
+			    echo "Skipping \"$arg\" (not found)!"
+			    continue
+			fi
+
+			target="`echo $arg | cut -f2 -d/`"
+
+			while read cksum file url flags ; do
+			    download_file "`source_file cksum $file url $flags`" \
+				"$url" "$cksum" "$target"
+			done < <(cat $arg)
+			;;
+		scripts/*)
+			: #FIXME miscdown.txt not handled yet
+			;;
+		*)
+			if [ ! "${arg%.desc}" = "$arg" ]; then
+			    descfile=$arg
+			else
+			    descfile="`echo package/*/$arg/$arg.desc`"
+			fi
+			
+			if [ ! -f $descfile ]; then
+			    echo "Skipping \"$arg\" (not found)!"
+			    continue
+			fi
+			
+			pkg="`echo $descfile | cut -f3 -d/`"
+			repo="`echo $descfile | cut -f2 -d/`"
+			
+			while read tag cksum file url flags ; do
+			    download_file "`source_file cksum $file url $flags`" \
+				"$url" "$cksum" "$repo" "$pkg"
+			done < <(grep -H '^\[D\] ' package/$repo/$pkg/$pkg.desc)
+			;;
+		esac
+	done
 }
 
 # Things to do only for downloading
@@ -812,14 +830,14 @@ case "$1" in
 	-list-missing)	list_missing ;;
 	-list-cksums)	list_cksums ;;
 
-	-pattern)	shift ; pattern "$@" ;;
-	-package)	shift ; package "$@" ;;
-	-repository)	shift ; repository "$@" ;;
 	-required)	required ;;
 	-all)		all ;;
 
+	-repository)	shift ; repository "$@" ;;
+
 	-*|"")		exec $0 --help ;;
-	*)		single_files "$@" ;;
+
+	*)		packages "$@" ;;
 esac
 
 exit 0

scripts/Puzzle

@@ -64,20 +64,15 @@ End
 EOF
 
 echo "Creating download/INDEX ..."
-mkdir -p download/
-{ echo "# This directory contains the package sources. They don't come"
-  echo "# with the rock-linux source distribution - so you need to download"
-  echo "# them. Type './scripts/Help Download' for details." ; echo "#"
+mkdir -p download/{mirror,nodist}
+{ 
+	echo "# This directory contains the package sources. They don't come"
+	echo "# with the rock-linux source distribution - so you need to download"
+	echo "# them. Type './scripts/Help Download' for details." ; echo "#"
 
-  {	grep -H '^\[D\] ' package/*/*/*.desc | tr '\t' ' ' | tr -s ' ' |
-                sed -e 's,^package/,download/,;'
-	grep -H '^[X0-9]' target/*/download.txt | tr '\t' ' ' | tr -s ' ' |
-                sed -e 's,^target/,download/,; s,:,:[D] ,;'
-	grep -H '^[X0-9]' scripts/miscdown.txt | tr '\t' ' ' | tr -s ' ' |
-                sed 's,^scripts/,download/misc/,; s,:,:[D] ,;'
-  } | sed 's,^\(.*/\)[^/:]*:[^ ]* \([X0-9]*\) ,\2 \1,;' |
-  tr ' ' '\t' | tr -s '\t' | sort -k2 |
-  awk '$1 !~ /^XX*$/ { printf("%010.0f %-50s %s\n", $1, $2, $3); }
-       $1  ~ /^XX*$/ { printf("XXXXXXXXXX %-50s %s\n",  $2, $3); }'
+	./scripts/Download -list-cksums |
+		tr ' ' '\t' | tr -s '\t' | sort -k2 |
+		awk '$1 !~ /^XX*$/ { printf("%010.0f %-50s %s\n", $1, $2, $3); }
+			$1  ~ /^XX*$/ { printf("XXXXXXXXXX %-50s %s\n",  $2, $3); }'
 } > download/INDEX
 

scripts/functions

@@ -129,6 +129,21 @@ copy_function() {
 	eval "$( declare -f $1 | sed "1 s,$1,$2," )"
 }
 
+# | column_clean |
+#
+# convert tabs to spaces, transform multiple consecutive spaces to one,
+# remove leading and trailing spaces
+column_clean() {
+	tr '\t' ' ' | tr -s ' ' | sed -e 's,^[ ]*,,; s,[ ]*$,,;'
+}
+
+# | column_clean_tab |
+#
+# see column_clean, but with tabs
+column_clean_tab() {
+	tr ' ' '\t' | tr -s '\t' | sed -e 's,^[\t]*,,; s,[\t]*$,,;'
+}
+
 # This function sets the 'confopt' and some other variables.
 # Re-run it in the package .conf file if you modify $prefix
 #
@@ -695,3 +710,34 @@ build_this_package() {
 
 	return 0
 }
+
+# [D] line regex:
+Dre='\([^:]*\):\([^ ]*\)[ ]\([^ ]*\)[ ]\(\([^ ]\)[^ ]*\)[ ]\([^ ]*\)'
+# \1: package-desc-path                 \2: '[D]'
+# \3: cksum                             \4: archive-file
+# \5: first-letter-of-archive-file      \6: download-url
+
+# NODIST flag regex:
+NODISTre='[nN][oO][dD][iI][sS][tT]'
+
+# source_file cksum file url flags...
+#
+# Create the file path from 'file' and 'flags'.
+# cksum and url are ignored
+# ([D] tag compatible format)
+source_file() {
+	local pre="" file="$2"
+	shift; shift; shift
+	
+	# inside Build-Pkg $archdir is set
+	if [ -n "$archdir" ]; then
+		pre=$base/; file="$( echo $file | sed 's,.\(t\?\)\(gz\|Z\)$,.\1bz2,' )"
+	fi
+    
+	if echo $* | egrep -q "($NODISTre)"; then 
+		echo ${pre}download/nodist/${file:0:1}/$file
+	else
+		echo ${pre}download/mirror/${file:0:1}/$file
+	fi
+}
+