From 08579ce683de69f5ba6aa173182f3981c26c6537 Mon Sep 17 00:00:00 2001 From: "Juergen \"George\" Sawinski" Date: Thu, 24 Jun 2004 11:13:26 +0000 Subject: [PATCH] Juergen Sawinski: New download/ hiarchy (see [2004052802365415769]) Clifford Wolf: Fixed building of packages with *.gz download URLs Fixed Build-Pkg $archdir creation (so it works, e.g. with tmpfs) [2004061412515820714] (https://www.rocklinux.net/submaster) git-svn-id: http://www.rocklinux.org/svn/rock-linux/trunk@3362 c5f82cb5-29bc-0310-9cd0-bff59a50e3bc --- scripts/Build-Pkg | 12 +- scripts/Create-CkSumPatch | 23 ++-- scripts/Download | 248 ++++++++++++++++++++------------------ scripts/Puzzle | 23 ++-- scripts/functions | 46 +++++++ 5 files changed, 204 insertions(+), 148 deletions(-) diff --git a/scripts/Build-Pkg b/scripts/Build-Pkg index 6e2f0b9ad..17eb924cb 100755 --- a/scripts/Build-Pkg +++ b/scripts/Build-Pkg @@ -246,8 +246,7 @@ if [ "$ROCKCFG_RETRY_BROKEN" = 0 -a $norebuild = 1 -a \ exit 1 fi -confdir="" ; archdir="" - +confdir="" if [ -z "$pkgdir" ] ; then for x in package/*/$pkg/$pkg.desc ; do if [ -f "$x" ] ; then @@ -259,14 +258,12 @@ if [ -z "$pkgdir" ] ; then fi x=${x#package/}; x=${x%%/*} confdir="$base/package/$x/$pkg" - archdir="$base/download/$x/$pkg" repository=$x fi done else if [ -f "$pkgdir/$pkg.desc" ] ; then confdir="$pkgdir" - archdir="$pkgdir" repository=extern fi fi @@ -572,6 +569,13 @@ elif [ "$ROCKCFG_FLIST" = "find" ] ; then sleep 2 fi +echo_status "Creating source archive directory." +archdir="$builddir/archdir"; mkdir -p "$archdir" +while read l_cksum l_srctar l_url l_flags ; do + [ -z "$l_cksum" ] && continue + ln -s `source_file l_cksum $l_srctar l_url $l_flags` $archdir/ +done < <( echo "$desc_D" | column_clean ) + hook_eval prepare # define new abort function for errors while building diff --git a/scripts/Create-CkSumPatch b/scripts/Create-CkSumPatch index d85fdf6d3..88010f6cb 100755 --- a/scripts/Create-CkSumPatch +++ b/scripts/Create-CkSumPatch @@ -52,10 +52,16 @@ while read cksum gzfile ; do if [ -f "$bzfile" ] then checkthis=1 - y="`echo $bzfile | cut -f$cmpfield -d/`" + + xfile="`basename $gzfile`" + descfile="`grep $xfile package/*/*/*.desc target/*/download.txt scripts/miscdown.txt | sed 's,:.*,,'`" + rep="`echo $descfile | cut -f2 -d/`" + pkg="`echo $descfile | cut -f3 -d/`" + for x ; do [ $checkthis = 1 ] && checkthis=0 - [ "$y" = $x ] && checkthis=2 + [ "$pkg" = $x ] && checkthis=2 + [ "$descfile" = $x ] && checkthis=2 done if [ $checkthis = 0 ] ; then : echo "Skip: $bzfile" ; continue @@ -70,13 +76,7 @@ while read cksum gzfile ; do fi echo $cksum >&2 - rep="`echo $bzfile | cut -f2 -d/`" - if [ -d package/$rep ] ; then - xfile="`echo $gzfile | cut -f4- -d/`" - pkgdir="package/`echo $gzfile | cut -f2,3 -d/`" - descfile="$pkgdir/`echo $gzfile | cut -f3 -d/`.desc" - if [ -f $descfile ] ; then sedscript="s,\[D\] *00* *$xfile,[D] $cksum $xfile," sed "$sedscript" < $descfile > /tmp/$$ @@ -93,13 +93,6 @@ while read cksum gzfile ; do echo "!!! (My guess was $descfile)" >&2 fi elif [ -d target/$rep -o "$rep" = misc ] ; then - if [ -d target/$rep ] ; then - descfile="target/$rep/download.txt" - else - descfile="scripts/miscdown.txt" - fi - xfile="`echo $gzfile | cut -f3- -d/`" - if [ -f $descfile ] ; then sedscript="s,^00* *$xfile,$cksum $xfile," sed "$sedscript" < $descfile > /tmp/$$ diff --git a/scripts/Download b/scripts/Download index a2f686a91..012526d1e 100755 --- a/scripts/Download +++ b/scripts/Download @@ -28,19 +28,17 @@ # definitions (optionally using a mirroring 'cache' server). # # This script also allows for checksum display/validation. -# -# Key to understanding this script is that all file lists get broken -# down to a single file download using ./scripts/Download filename. umask 022 +. scripts/functions + if [ "$1" = '--help' ] ; then { echo echo "Usage:" echo - echo " ./scripts/Download [options] [ Filename(s) ]" - echo " ./scripts/Download [options] -pattern Pattern(s)" - echo " ./scripts/Download [options] -package Package(s)" + echo " ./scripts/Download [options] [ Package(s) ]" + echo " ./scripts/Download [options] [ Desc file(s) ]" echo " ./scripts/Download [options] -repository Repositories" echo " ./scripts/Download [options] { -all | -required }" echo @@ -49,6 +47,7 @@ if [ "$1" = '--help' ] ; then echo " [ -mirror | -check ] [ -try-questionable ] [ -notimeout ]" echo " [ -longtimeout ] [ -curl-opt [:[:..]] ]" echo " [ -proxy [:] ] [ -proxy-auth [:] ]" + echo " [ -copy ] [ -move ]" echo echo " On default, this script auto-detects the best ROCK Linux mirror." echo @@ -93,6 +92,8 @@ this_is_the_2nd_run=0 mirror='' ; checkonly=0 ; altdir='' ; loop=1 tryques=0 ; nocheck=0 ; options='-this_is_the_2nd_run ' notimeout=0 ; curl_options='--disable-epsv --location' +altcopy=link + # while [ $loop -eq 1 ] ; do case "$1" in @@ -176,6 +177,9 @@ while [ $loop -eq 1 ] ; do # also try to download questionable URLs options="$options -try-questionable" tryques=1 ; shift ;; + + -move) altcopy=move ; shift ;; + -copy) altcopy=copy ; shift ;; *) loop=0 ;; @@ -260,7 +264,7 @@ detect_mirror() { fi } -# download_file local-filename download-location cksum +# download_file local-filename download-location cksum repo pkg # # This function decides if download directly or from a mirror, # validates checksum, etc. @@ -270,7 +274,7 @@ download_file() { # Init # - gzfile="$1" ; location="$2" ; cksum="$3" + local gzfile="$1" location="$2" cksum="$3" repo="$4" pkg="$5" # Make src directory for creating tar balls mkdir -p src/ # Tarball file name: (if you change this one - also adapt Create-ISO) @@ -370,10 +374,25 @@ download_file() { altfile="" fi + #FIXME: compatibility, can be removed sooner or later... + # Check old download dir layout + if [ -z "$altfile" ]; then + if [ -f "download/$repo${pkg:+/}$pkg/`basename $bzfile`" ]; then + altfile="download/$repo${pkg:+/}$pkg/`basename $bzfile`" + fi + fi + if [ "$altfile" ] ; then echo "Found `basename $bzfile` as $altfile." - cp -lv $altfile $bzfile ; gzfile="$bzfile" + if [ "$altcopy" = 'link' ]; then + cp -lv $altfile $bzfile + elif [ "$altcopy" = 'copy' ]; then + cp -v $altfile $bzfile + elif [ "$altcopy" = 'move' ]; then + mv -v $altfile $bzfile + fi + gzfile="$bzfile" else @@ -383,10 +402,15 @@ download_file() { if [ "$mirror" -a "$mirror" != "none" ] ; then # try to use mirror + if ! download_file_now "!$mirror/${bzfile#download/}" $bzfile $bzfile; then - # oops... so try direct (original URL) - echo "INFO: download from mirror failed, trying original URL." - download_file_now $location $gzfile $bzfile + #FIXME: compatibility, can be removed sooner or later... + # Try old download format for remote file + if ! download_file_now "!$mirror/$repo${pkg:+/}$pkg/`basename $bzfile`" $bzfile $bzfile; then + # oops... so try direct (original URL) + echo "INFO: download from mirror failed, trying original URL." + download_file_now $location $gzfile $bzfile + fi else gzfile="$bzfile" fi @@ -456,7 +480,8 @@ download_file_now() { if test -s src/Download-Errors && grep -q " $url\$" src/Download-Errors ; then echo "ERROR: According to src/Download-Errors" \ - "we had already an error for that URL." + "we had already an error for the URL" + echo "ERROR: $url" echo "ERROR: So I'm not trying to download" \ "it again (remove src/Download-Errors" echo "ERROR: if you want to force a retry)." @@ -601,69 +626,25 @@ download_file_now() { fi } +list_dtags() { + { + grep -H '^\[D\] ' package/*/*/*.desc + grep -H '^[X0-9]' target/*/download.txt | sed 's,:,:[D] ,' -# handle_file filename -# -# This function fetches the checksum and download information -# from the desc files and calls the download_file function. -# -handle_file() { - - # Handle pkg file (package/.../*.desc) - # - tree="`echo "$1" | cut -f2 -d/`" - pkg="` echo "$1" | cut -f3 -d/`" - file="`echo "$1" | cut -f4 -d/`" - if grep -q "^\[D\].* $file " package/$tree/$pkg/$pkg.desc 2> /dev/null - then - grep "^\[D\].* $file " package/$tree/$pkg/$pkg.desc | - while read tag cksum file url ; do - download_file "$1" "$url" "$cksum" - done - return 0 - fi - - # Handle target file (target/.../download.txt) - # - target="`echo "$1" | cut -f2 -d/`" - file="`echo "$1" | cut -f3- -d/`" - if grep -q " $file " target/$target/download.txt 2> /dev/null - then - grep " $file " target/$target/download.txt | - while read cksum file url ; do - download_file "$1" "$url" "$cksum" - done - return 0 - fi - - # Handle misc file (scripts/miscdown.txt) - # - if [ "${1#download/misc/}" != "$1" ] ; then - file="`echo "$1" | cut -f3- -d/`" - if grep -q " $file " scripts/miscdown.txt - then - grep " $file " scripts/miscdown.txt | - while read cksum file url ; do - download_file "$1" "$url" "$cksum" - done - return 0 - fi - fi - - # Can't handle file / file not found - # - return 1 + #FIXME the following have relative paths in the archive-file column + # is it needed anyway? + grep -H '^[X0-9]' scripts/miscdown.txt | sed 's,:,:[D] ,' + } | column_clean } list_cksums() { trap '' INT - { grep -H '^\[D\] ' package/*/*/*.desc | tr '\t' ' ' | tr -s ' ' | - sed -e 's,^package/,download/,;' - grep -H '^[X0-9]' target/*/download.txt | tr '\t' ' ' | tr -s ' ' | - sed -e 's,^target/,download/,; s,:,:[D] ,;' - grep -H '^[X0-9]' scripts/miscdown.txt | tr '\t' ' ' | tr -s ' ' | - sed 's,^scripts/,download/misc/,; s,:,:[D] ,;' - } | sed 's,^\(.*/\)[^/:]*:[^ ]* \([X0-9]*\) ,\2 \1,;' | cut -f1,2 -d' ' + + list_dtags | sed \ + -e "s,^$Dre[ ].*\($NODISTre\).*$,\3 download/nodist/\5/\4,;" \ + -e "s,^$Dre$,\3 download/mirror/\5/\4,;" \ + | sed 's,^\(.*/\)[^/:]*:[^ ]* \([X0-9]*\) ,\2 \1,;' | cut -f1,2 -d' ' + trap INT } @@ -696,27 +677,9 @@ list_missing() { trap INT } -pattern() { - for pattern ; do - echo "Processing pattern $pattern ..." - list | while read fn ; do - if [ "${fn##$pattern}" = '' ] ; then - echo "Matched: $fn" - $0 $options "$fn" - fi - done - done -} - -package() { - for package ; do - pattern "download/*/$package/*" - done -} - repository() { for repository ; do - pattern "download/$repository/*" + packages `echo package/$repository/*/*.desc` done } @@ -729,41 +692,96 @@ required() { exit 1 fi - while read on a b tree pkg c ; do + while read on a b repo pkg c ; do if [ "$on" = "X" ] ; then - grep -H '^\[D\] ' package/$tree/$pkg/$pkg.desc > src/down.$$.lst - while read tag cksum file url ; do - download_file "download/$tree/$pkg/$file" "$url" "$cksum" + grep -H '^\[D\] ' package/$repo/$pkg/$pkg.desc > src/down.$$.lst + while read tag cksum file url flags ; do + download_file "`source_file cksum $file url $flags`" "$url" "$cksum" "$repo" "$pkg" done < src/down.$$.lst ; rm -f src/down.$$.lst fi done < config/$config/packages target=`grep '^export ROCKCFG_TARGET=' config/$config/config | \ cut -f2 -d= | tr -d "'"` if [ -f target/$target/download.txt ] ; then - while read cksum file url ; do - download_file "download/$target/$file" "$url" "$cksum" + while read cksum file url flags ; do + download_file "`source_file cksum $file url $flags`" "$url" "$cksum" "$target" done < target/$target/download.txt fi + + #FIXME what's about scripts/miscdown.txt? } -single_files() { - for file_name ; do - file_name_gz="`echo "$file_name" | sed 's,\.\(t\?\)bz2$,.\1gz,'`" - file_name_Z="`echo "$file_name" | sed 's,\.\(t\?\)bz2$,.\1Z,'`" - if ! handle_file "$file_name" && \ - ! handle_file "$file_name_gz" && \ - ! handle_file "$file_name_Z" - then - echo "ERROR: Unknown file: $file_name." - fi +all() { + local each repo pkg + + for each in package/*/*/*.desc; do + pkg="`echo $descfile | cut -f3 -d/`" + repo="`echo $descfile | cut -f2 -d/`" + + while read tag cksum file url flags ; do + download_file "`source_file cksum $file url $flags`" "$url" "$cksum" "$repo" "$pkg" + done < <(grep -H '^\[D\] ' package/$repo/$pkg/$pkg.desc) + done + + for each in target/*/download.txt; do + target="`echo $descfile | cut -f2 -d/`" + + while read cksum file url flags ; do + download_file "`source_file cksum $file url $flags`" "$url" "$cksum" "$target" + done < <(cat $each) + done + + #FIXME miscdown.txt contains relative paths in 2nd column + for each in scripts/miscdown.txt; do + while read cksum file url flags ; do + file="`basename $file`" + download_file "`source_file cksum $file url $flags`" "$url" "$cksum" "$target" + done < <(cat $each) done } -all() { - if [ $checkonly = 1 ] ; then list - else list_missing ; fi > src/down.$$.lst - while read fn ; do single_files $fn ; done < src/down.$$.lst - rm -f src/down.$$.lst +packages() { + local descfile + for arg; do + case "$arg" in + target/*) + if [ ! -f $arg ]; then + echo "Skipping \"$arg\" (not found)!" + continue + fi + + target="`echo $arg | cut -f2 -d/`" + + while read cksum file url flags ; do + download_file "`source_file cksum $file url $flags`" \ + "$url" "$cksum" "$target" + done < <(cat $arg) + ;; + scripts/*) + : #FIXME miscdown.txt not handled yet + ;; + *) + if [ ! "${arg%.desc}" = "$arg" ]; then + descfile=$arg + else + descfile="`echo package/*/$arg/$arg.desc`" + fi + + if [ ! -f $descfile ]; then + echo "Skipping \"$arg\" (not found)!" + continue + fi + + pkg="`echo $descfile | cut -f3 -d/`" + repo="`echo $descfile | cut -f2 -d/`" + + while read tag cksum file url flags ; do + download_file "`source_file cksum $file url $flags`" \ + "$url" "$cksum" "$repo" "$pkg" + done < <(grep -H '^\[D\] ' package/$repo/$pkg/$pkg.desc) + ;; + esac + done } # Things to do only for downloading @@ -812,14 +830,14 @@ case "$1" in -list-missing) list_missing ;; -list-cksums) list_cksums ;; - -pattern) shift ; pattern "$@" ;; - -package) shift ; package "$@" ;; - -repository) shift ; repository "$@" ;; -required) required ;; -all) all ;; + -repository) shift ; repository "$@" ;; + -*|"") exec $0 --help ;; - *) single_files "$@" ;; + + *) packages "$@" ;; esac exit 0 diff --git a/scripts/Puzzle b/scripts/Puzzle index e0ad9d93b..d34aed3d6 100755 --- a/scripts/Puzzle +++ b/scripts/Puzzle @@ -64,20 +64,15 @@ End EOF echo "Creating download/INDEX ..." -mkdir -p download/ -{ echo "# This directory contains the package sources. They don't come" - echo "# with the rock-linux source distribution - so you need to download" - echo "# them. Type './scripts/Help Download' for details." ; echo "#" +mkdir -p download/{mirror,nodist} +{ + echo "# This directory contains the package sources. They don't come" + echo "# with the rock-linux source distribution - so you need to download" + echo "# them. Type './scripts/Help Download' for details." ; echo "#" - { grep -H '^\[D\] ' package/*/*/*.desc | tr '\t' ' ' | tr -s ' ' | - sed -e 's,^package/,download/,;' - grep -H '^[X0-9]' target/*/download.txt | tr '\t' ' ' | tr -s ' ' | - sed -e 's,^target/,download/,; s,:,:[D] ,;' - grep -H '^[X0-9]' scripts/miscdown.txt | tr '\t' ' ' | tr -s ' ' | - sed 's,^scripts/,download/misc/,; s,:,:[D] ,;' - } | sed 's,^\(.*/\)[^/:]*:[^ ]* \([X0-9]*\) ,\2 \1,;' | - tr ' ' '\t' | tr -s '\t' | sort -k2 | - awk '$1 !~ /^XX*$/ { printf("%010.0f %-50s %s\n", $1, $2, $3); } - $1 ~ /^XX*$/ { printf("XXXXXXXXXX %-50s %s\n", $2, $3); }' + ./scripts/Download -list-cksums | + tr ' ' '\t' | tr -s '\t' | sort -k2 | + awk '$1 !~ /^XX*$/ { printf("%010.0f %-50s %s\n", $1, $2, $3); } + $1 ~ /^XX*$/ { printf("XXXXXXXXXX %-50s %s\n", $2, $3); }' } > download/INDEX diff --git a/scripts/functions b/scripts/functions index 74fbd2dce..3bf7836d6 100644 --- a/scripts/functions +++ b/scripts/functions @@ -129,6 +129,21 @@ copy_function() { eval "$( declare -f $1 | sed "1 s,$1,$2," )" } +# | column_clean | +# +# convert tabs to spaces, transform multiple consecutive spaces to one, +# remove leading and trailing spaces +column_clean() { + tr '\t' ' ' | tr -s ' ' | sed -e 's,^[ ]*,,; s,[ ]*$,,;' +} + +# | column_clean_tab | +# +# see column_clean, but with tabs +column_clean_tab() { + tr ' ' '\t' | tr -s '\t' | sed -e 's,^[\t]*,,; s,[\t]*$,,;' +} + # This function sets the 'confopt' and some other variables. # Re-run it in the package .conf file if you modify $prefix # @@ -695,3 +710,34 @@ build_this_package() { return 0 } + +# [D] line regex: +Dre='\([^:]*\):\([^ ]*\)[ ]\([^ ]*\)[ ]\(\([^ ]\)[^ ]*\)[ ]\([^ ]*\)' +# \1: package-desc-path \2: '[D]' +# \3: cksum \4: archive-file +# \5: first-letter-of-archive-file \6: download-url + +# NODIST flag regex: +NODISTre='[nN][oO][dD][iI][sS][tT]' + +# source_file cksum file url flags... +# +# Create the file path from 'file' and 'flags'. +# cksum and url are ignored +# ([D] tag compatible format) +source_file() { + local pre="" file="$2" + shift; shift; shift + + # inside Build-Pkg $archdir is set + if [ -n "$archdir" ]; then + pre=$base/; file="$( echo $file | sed 's,.\(t\?\)\(gz\|Z\)$,.\1bz2,' )" + fi + + if echo $* | egrep -q "($NODISTre)"; then + echo ${pre}download/nodist/${file:0:1}/$file + else + echo ${pre}download/mirror/${file:0:1}/$file + fi +} +