OpenSDE Framework (without history before r20070)
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

817 lines
20 KiB

  1. #!/bin/bash
  2. # --- SDE-COPYRIGHT-NOTE-BEGIN ---
  3. # This copyright note is auto-generated by ./scripts/Create-CopyPatch.
  4. #
  5. # Filename: bin/sde-download
  6. # Copyright (C) 2006 - 2007 The OpenSDE Project
  7. # Copyright (C) 2004 - 2006 The T2 SDE Project
  8. # Copyright (C) 1998 - 2003 Clifford Wolf
  9. #
  10. # More information can be found in the files COPYING and README.
  11. #
  12. # This program is free software; you can redistribute it and/or modify
  13. # it under the terms of the GNU General Public License as published by
  14. # the Free Software Foundation; version 2 of the License. A copy of the
  15. # GNU General Public License can be found in the file COPYING.
  16. # --- SDE-COPYRIGHT-NOTE-END ---
  17. #Description: Download sources
  18. #Alias: get
  19. set -e
  20. [ -n "$SDEROOT" ] ||
  21. export SDEROOT=$( cd "${0%/*}/.."; pwd -P )
  22. . $SDEROOT/lib/libsde.in
  23. . $SDEROOT/lib/libsde-ini.in # to write
  24. . $SDEROOT/lib/functions.in
  25. umask 022
  26. cd "$SDEROOT"
  27. # -mk-cksum mode (display OpenSDE type package checksum): it
  28. # displays the checksum OpenSDE validates against.
  29. #
  30. # Currently bz2, tbz2, gz, tgz, Z are unpacked
  31. #
  32. if [ "$1" = -mk-cksum ] ; then
  33. shift
  34. exec sh ./lib/sde-download/cksum.sh "$@"
  35. fi
  36. # Handle options passed on the command line
  37. #
  38. mkdir -p tmp/ download/
  39. # Load system wide configuration for this tool
  40. #
  41. config=default mirror= altdir= proxy= proxyauth=
  42. if [ -s "$SDESETTINGS" ]; then
  43. eval $( ini_section_read "$SDESETTINGS" download )
  44. eval $( ini_section_read "$SDESETTINGS" download-$sdever )
  45. elif [ -s "$HOME/.sde/download" ]; then
  46. # import old $HOME/.sde/download
  47. . "$HOME/.sde/download"
  48. ini_write "$SDESETTINGS" 'download' altdir
  49. fi
  50. # deprecation of $HOME/.sde/download
  51. if [ -f "$HOME/.sde/download" ]; then
  52. echo_warning "\$HOME/.sde/download is deprecated." 1>&2
  53. fi
  54. this_is_the_2nd_run=0
  55. checkonly=0 tryques=0 nocheck=0 notimeout=0
  56. options='-this_is_the_2nd_run '
  57. curl_options='-A opensde-downloader --disable-epsv --location -f'
  58. altcopy=link ; verbose=1
  59. downloaderror=0
  60. #
  61. while [ $# -gt 0 ]; do
  62. case "$1" in
  63. -this_is_the_2nd_run)
  64. this_is_the_2nd_run=1
  65. ;;
  66. -cfg)
  67. options="$options -cfg $2"
  68. config="$2" ; shift ;;
  69. -q)
  70. options="$options -q"
  71. verbose=0 ;;
  72. -nock)
  73. # -nock skips checksum checking (don't use lightly)
  74. options="$options -nock"
  75. nocheck=1 ; shift ;;
  76. -mirror)
  77. # -mirror uses a mirror for finding source files
  78. if [ "$2" = none ]; then
  79. echo
  80. echo "The option '-mirror none' is not supported anymore!"
  81. echo
  82. echo "You may edit \$HOME/.sde/settings if you really"
  83. echo "want to use the original download resources. However, this"
  84. echo "is not supported and if such a download fails, this is not"
  85. echo "a bug in the OpenSDE and doesn't neccessarily needs fixing."
  86. echo
  87. exit 1;
  88. elif [ "$2" = auto ]; then
  89. mirror=
  90. else
  91. options="$options -mirror $2"
  92. mirror="$2"
  93. ini_write "$SDESETTINGS" download-$sdever mirror
  94. fi
  95. shift ;;
  96. -check)
  97. # -check just validates the file using the checksum
  98. options="$options -check"
  99. checkonly=1 ;;
  100. -notimeout)
  101. # don't add timeout curl options
  102. options="$options -notimeout"
  103. notimeout=2 ;;
  104. -longtimeout)
  105. # don't add timeout curl options
  106. options="$options -longtimeout"
  107. notimeout=1 ;;
  108. -curl-opt)
  109. # additional curl options
  110. options="$options -curl-opt $2"
  111. curl_options="$curl_options `echo $2 | tr : ' '`"
  112. shift ;;
  113. -alt-dir)
  114. # check for an alternative directory where to search for
  115. # package source tarballs
  116. altdir=$( cd $2; pwd -P )
  117. options="$options -alt-dir $2"
  118. shift ;;
  119. -try-questionable)
  120. # also try to download questionable URLs
  121. options="$options -try-questionable"
  122. tryques=1 ;;
  123. -move) altcopy=move ;;
  124. -copy) altcopy=copy ;;
  125. *) break ;;
  126. esac
  127. shift
  128. done
  129. if [ $notimeout -eq 0 ] ; then
  130. curl_options="$curl_options -y 10 -Y 10 --connect-timeout 60"
  131. fi
  132. if [ $notimeout -eq 1 ] ; then
  133. curl_options="$curl_options -y 60 -Y 1 --connect-timeout 300"
  134. fi
  135. # proxy (server[:port])
  136. if [ -n "$proxy" ]; then
  137. curl_options="$curl_options --proxy $proxy"
  138. # proxy needs auth (username[:password])
  139. [ -z "$proxyauth" ] || curl_options="$curl_options --proxy-user $proxyauth"
  140. # only show once
  141. [ $this_is_the_2nd_run = 1 ] || echo_info "Using <$proxy> as ${proxyauth:+authenticated }http proxy."
  142. fi
  143. #Disable checking for certificates on https downloads
  144. curl_options="$curl_options -k"
  145. # Autodetect best Mirror and safe url in $mirror
  146. #
  147. detect_mirror() {
  148. if [ "$mirror" = "none" ] ; then
  149. echo_info "Using original download locations only."
  150. return
  151. elif [ "$mirror" = "broken" ]; then
  152. echo_warning "Previous detection of the mirror failed, trying again."
  153. elif [ -n "$mirror" ]; then
  154. echo_info "Using mirror <$mirror>."
  155. return
  156. fi
  157. echo_warning "Auto-detecting best mirror ..."
  158. echo_info "Downloading mirror-list from opensde.net."
  159. curl -s -S $curl_options -o tmp/Download-Mirror-List \
  160. "http://opensde.net/opensde-download-mirrors/$sdever"
  161. if [ -r tmp/Download-Mirror-List ]; then
  162. bash lib/sde-download/mirror-test.sh < tmp/Download-Mirror-List
  163. fi 2>&1 | echo_info
  164. # read new mirror info
  165. eval $( ini_section_read "$SDESETTINGS" download-$sdever )
  166. if [ "$mirror" == "broken" ]; then
  167. echo_warning "No Mirror Found!"
  168. else
  169. echo_info "Using mirror <$mirror>."
  170. fi
  171. }
  172. # download_file local-filename download-location cksum repo pkg
  173. #
  174. # This function decides if download directly or from a mirror,
  175. # validates checksum, etc.
  176. # Calls download_file_now to do the actual download.
  177. #
  178. download_file() {
  179. # Init
  180. #
  181. local gzfile="$1" location="$2" cksum="$3" repo="$4" pkg="$5"
  182. # Make src directory for creating tar balls
  183. mkdir -p tmp/
  184. # Tarball file name:
  185. bzfile="`bz2filename "$gzfile"`"
  186. # Remove optional '-' prefix from $location
  187. [ "${location:0:1}" == '-' ] && location="${location:1}"
  188. # Lock file name:
  189. lkfile="tmp/down.lockfile.`echo $bzfile | tr / -`"
  190. # Check if it's already there
  191. #
  192. [ -s "$bzfile" -a $checkonly != 1 ] && return 0
  193. # Make locking
  194. #
  195. if [ -s "$lkfile" ]; then
  196. echo "Found $lkfile -> skip download."
  197. return 0
  198. fi
  199. trap 'rm -f "$lkfile"' INT
  200. echo $$ > "$lkfile"
  201. # Check if we only like to test the cksum(s)
  202. #
  203. if [ $checkonly = 1 ] ; then
  204. gzfile="$bzfile"
  205. if [ ! -f "$bzfile" ] ; then
  206. echo "File missing: $bzfile"
  207. rm -f "$lkfile" ; trap INT ; return 1
  208. fi
  209. if [ -z "${cksum##X*}" ] ; then
  210. echo "No checksum (ignore): $bzfile"
  211. rm -f "$lkfile" ; trap INT ; return 1
  212. fi
  213. if [ "$cksum" -eq 0 ] ; then
  214. echo "No checksum (missing): $bzfile"
  215. rm -f "$lkfile" ; trap INT ; return 1
  216. fi
  217. elif [ -s "$gzfile" ] ; then
  218. echo ; echo "Already downloaded $pkg:$gzfile ..."
  219. else
  220. echo ; echo "Downloading $pkg:$gzfile ..."
  221. # Existing *.cksum-err
  222. #
  223. if [ -s "$gzfile.cksum-err" ] ; then
  224. # cksum-err file alread exists:
  225. echo "ERROR: Found $gzfile.cksum-err."
  226. echo "ERROR: That means that we downloaded the" \
  227. "file already and it had an"
  228. echo "ERROR: incorrect checksum. Remove the" \
  229. "*.cksum-err file to force a"
  230. echo "ERROR: new download of that file."
  231. rm -f "$lkfile" ; trap INT ; return 1
  232. fi
  233. # Existing *.extck-err
  234. #
  235. if [ -s "$gzfile.extck-err" ] ; then
  236. # extck-err file alread exists:
  237. echo "ERROR: Found $gzfile.extck-err."
  238. echo "ERROR: That means that we downloaded the" \
  239. "file already and it's content"
  240. echo "ERROR: did not match it's filename extension." \
  241. "Remove the *.extck-err file"
  242. echo "ERROR: to force a new download of that file."
  243. rm -f "$lkfile" ; trap INT ; return 1
  244. fi
  245. # Questionable URL
  246. #
  247. if [ "$location" != "${location#\?}" ] ; then
  248. if [ "$tryques" = 0 ] ; then
  249. echo "ERROR: URL is marked as questionable." \
  250. "Not downloading this file."
  251. rm -f "$lkfile" ; trap INT ; return 1
  252. else
  253. echo "WARNING: URL is marked as questionable." \
  254. "Downloading it anyways."
  255. location="${location#\?}"
  256. fi
  257. fi
  258. # Make directory (if required)
  259. #
  260. if [ ! -d `dirname "$bzfile"` ] ; then
  261. mkdir -p `dirname "$bzfile"`
  262. fi
  263. # Alternative Directory
  264. #
  265. if [ "$altdir" ] ; then
  266. altfile=$(find $altdir/ -name `basename $bzfile` | head -n 1)
  267. else
  268. altfile=""
  269. fi
  270. #FIXME: compatibility, can be removed sooner or later...
  271. # Check old download dir layout
  272. if [ -z "$altfile" ]; then
  273. if [ -f "download/$repo${pkg:+/}$pkg/`basename $bzfile`" ]; then
  274. altfile="download/$repo${pkg:+/}$pkg/`basename $bzfile`"
  275. fi
  276. fi
  277. if [ "$altfile" ] ; then
  278. echo "Found `basename $bzfile` as $altfile."
  279. if [ "$altcopy" = 'link' ]; then
  280. cp -lv $altfile $bzfile
  281. elif [ "$altcopy" = 'copy' ]; then
  282. cp -v $altfile $bzfile
  283. elif [ "$altcopy" = 'move' ]; then
  284. mv -v $altfile $bzfile
  285. fi
  286. gzfile="$bzfile"
  287. else
  288. # Mirroring
  289. #
  290. if [ -n "$mirror" -a "$mirror" != "none" -a "$mirror" != "broken" -a -z "${bzfile##download/mirror/*}" ] ; then
  291. # try to use mirror
  292. if ! download_file_now "!$mirror/${bzfile#download/mirror/}" $bzfile $bzfile; then
  293. echo "INFO: download from mirror failed, trying original URL."
  294. download_file_now "$location" $gzfile $bzfile \
  295. || downloaderror=1
  296. else
  297. gzfile="$bzfile"
  298. fi
  299. else
  300. # don't want to use mirror
  301. download_file_now "$location" $gzfile $bzfile \
  302. downloaderror=1
  303. fi
  304. fi
  305. if [ ! -s "$gzfile" ]; then
  306. rm -f "$lkfile" ; trap INT ; return 1
  307. fi
  308. fi
  309. # unsign .gpg file
  310. if [[ $gzfile = *.gpg ]]; then
  311. gzfile=${gzfile%.gpg}
  312. if [ -f $gzfile.gpg ]; then
  313. echo "unsigning GnuPG file: $gzfile.gpg"
  314. gpg $gzfile.gpg
  315. fi
  316. if [ ! -f $gzfile ]; then
  317. echo "unsigning failed"
  318. rm -f "$lkfile" ; trap INT ; return 1
  319. fi
  320. fi
  321. sh ./lib/sde-download/validate.sh "$gzfile" "$bzfile" "$cksum" || downloaderror=1
  322. # Free Lock and finish
  323. #
  324. rm -f "$lkfile" ; trap INT ; return 0
  325. }
  326. # download_file_now location remote_filename local_filename
  327. #
  328. # This function executes the actual download using curl.
  329. #
  330. download_file_now() {
  331. local location="$1" gzfile="$2" bzfile="$3" curlret=0
  332. # Create URL
  333. #
  334. case "$location" in
  335. manual://*) url="$location" ;;
  336. !*) url="${location#!}" ;;
  337. *) url="${location%/*}/${gzfile##*/}" ;;
  338. esac
  339. # Download
  340. #
  341. case "$url" in
  342. manual://*)
  343. # Determine if the file has already been downloaded
  344. # manually. For this we first look in $HOME then in
  345. # download/manual.
  346. downloadpath=${altdir:-$HOME}
  347. downloadfile="${gzfile##*/}"
  348. if [ -e $downloadpath/$downloadfile ]; then
  349. location="file://$downloadpath/"
  350. else
  351. location="http://${url#manual://}"
  352. # No manual download has taken place yet.
  353. # So inform the user to do so.
  354. cat <<-EOT
  355. The file $downloadfile can not be fetched automatically
  356. please visit: $location
  357. and download it manually into $HOME or somewhere else using -alt-dir
  358. EOT
  359. return 1;
  360. fi
  361. # I am to lazy to do the copy and conversion myself,
  362. # so I use this function again with a modified
  363. # download location.
  364. download_file_now "$location" $gzfile $bzfile
  365. return "$?"
  366. ;;
  367. http://*|https://*|ftp://*|file://*)
  368. if [ -s "$gzfile.incomplete" ] ; then
  369. echo "INFO: Trying to resume previous download .."
  370. resume="-C -"
  371. else
  372. resume=""
  373. fi
  374. [ -s download/translations.sed ] &&
  375. trfile=download/translations.sh ||
  376. trfile=etc/download.sed
  377. trurl="$( echo "$url" | sed -f $trfile )"
  378. if [ -n "$trurl" -a "$trurl" != "$url" ]; then
  379. echo "INFO: url translated."
  380. url="$trurl"
  381. fi
  382. unset trurl trfile
  383. curl -w '\rFinished downloading %{size_download} bytes in %{time_total} seconds (%{speed_download} bytes/sec). \n' --progress-bar $resume $curl_options "$url" -o "$gzfile.incomplete"
  384. curlret="$?"
  385. if [ "$resume" ] && \
  386. [ $curlret -eq 33 -o $curlret -eq 36 ] ; then
  387. echo "INFO: Resuming download not possible. ->" \
  388. "Overwriting old file."
  389. rm -f "$gzfile.incomplete"
  390. curl -w '\rFinished downloading %{size_download} bytes in %{time_total} seconds (%{speed_download} bytes/sec). \n' --progress-bar $curl_options "$url" -o "$gzfile.incomplete"
  391. curlret="$?"
  392. fi
  393. if [ $curlret -ne 0 ] ; then
  394. case "$curlret" in
  395. 18)
  396. echo "WARNING: Got only some of the" \
  397. "file. A re-run of $0"
  398. echo "WARNING: is required to complete" \
  399. "the download." ;;
  400. 130)
  401. echo -e '\rWARNING: CURL got a SIGINT' \
  402. "(someone pressed Ctrl-C). A re-run of"
  403. echo "WARNING: $0 is required to complete" \
  404. "the download." ; sleep 1 ;;
  405. *)
  406. echo "$curlret $gzfile $url" \
  407. >> tmp/Download-Errors
  408. echo -e '\rERROR: CURL Returned Error' \
  409. "$curlret. Please read" \
  410. "the curl manpage." ;;
  411. esac
  412. return 1
  413. elif [ ! -s "$gzfile.incomplete" ] ; then
  414. echo "0 $gzfile $url" >> tmp/Download-Errors
  415. echo "ERROR: CURL returned success but" \
  416. "we have no data!"
  417. curlret=1
  418. else
  419. case "$gzfile" in
  420. *.gz|*.tgz)
  421. typeexpr="gzip compressed data" ;;
  422. *.bz2|*.tbz2|*.tbz)
  423. typeexpr="bzip2 compressed data" ;;
  424. *.Z|*.tZ)
  425. typeexpr="compress'd data" ;;
  426. *.zip|*.jar)
  427. typeexpr="Zip archive data" ;;
  428. *.tar)
  429. typeexpr="tar archive" ;;
  430. *)
  431. echo "WARNING: Unkown file extension: $gzfile"
  432. typeexpr="." ;;
  433. esac
  434. if file "$gzfile.incomplete" | grep -v "$typeexpr"
  435. then
  436. echo "ERROR: File type does not match" \
  437. "filename ($typeexpr)!"
  438. mv "$gzfile.incomplete" "$gzfile.extck-err"
  439. else
  440. mv "$gzfile.incomplete" "$gzfile"
  441. fi
  442. fi
  443. ;;
  444. *)
  445. protocol="${url%%://*}"
  446. # we need to use $location - $url is already mangled above -ReneR
  447. # $protocol://$url $options
  448. url="`echo "$location" | sed "s,$protocol://\([^ ]*\).*,\1,"`"
  449. options="`echo "$location" | cut -d' ' -f2-`"
  450. case "$protocol" in
  451. cvs)
  452. # the first option is the module name
  453. module="${options%% *}"
  454. options="${options#* }"
  455. cmdline="cvs -z4 -Q -d $url co -P $options $module"
  456. # sometimes cvs wants to read ~/.cvspass just for fun ..
  457. touch $HOME/.cvspass
  458. ;;
  459. svn|svn\+http)
  460. if [ "$protocol" == "svn+http" ]; then
  461. url="http://$url"
  462. else
  463. url="svn://$url"
  464. fi
  465. if [ "${options:0:1}" == "-" ]; then
  466. # the module is the last dir of $url
  467. module="${url##*/}"
  468. else
  469. # the first option is the module name
  470. module="${options%% *}"
  471. options="${options#* }"
  472. fi
  473. cmdline="svn co $options $url $module"
  474. ;;
  475. *)
  476. echo "$cmdclient unrecognized!"
  477. return 1
  478. ;;
  479. esac
  480. cvsdir="tmp/down.${protocol}dir.`echo $bzfile | tr / -`"
  481. saved_pwd=$PWD ; mkdir -p $cvsdir ; cd $cvsdir
  482. echo "$cmdline"
  483. {
  484. $cmdline || touch .cvs_error
  485. } &> .cvs_output &
  486. while fuser .cvs_output &> /dev/null ; do
  487. echo -ne `nice du -sh 2> /dev/null | \
  488. cut -f1` 'downloaded from archive so far...\r'
  489. sleep 3
  490. done
  491. if [ -f .cvs_error ] ; then
  492. cd $saved_pwd ; rm -rf $cvsdir
  493. echo -e "\nError during checkout."
  494. return 1
  495. fi
  496. echo `du -sh 2> /dev/null | \
  497. cut -f1` 'downloaded from archive (download finished).'
  498. if [ `echo * | wc -w` -gt 1 ]; then
  499. # multi-module module
  500. echo "Multi-module package detected, relocating..."
  501. mkdir t2-module.$$
  502. for x in *; do
  503. [ "$x" != "t2-module.$$" ] && mv -f $x t2-module.$$/
  504. done
  505. mkdir -p "$module"
  506. mv -f t2-module.$$/* "$module"
  507. rm -f t2-module.$$
  508. fi
  509. cd `dirname $module`
  510. tarname="`basename $bzfile`"
  511. echo "Preparing files for final tarball ..."
  512. find -type d \( -name CVS -o -name .svn \) | xargs rm -rf
  513. if [ `find -type f | wc -l` -gt 4 ]; then
  514. find `basename $module` | xargs touch -t 200001010000
  515. tar --owner root --group root \
  516. --use-compress-program=bzip2 \
  517. -cf $tarname `basename $module`
  518. mv $tarname $saved_pwd/$bzfile
  519. else
  520. echo "Too few files - assuming checkout failure."
  521. curlret=1
  522. fi
  523. cd $saved_pwd ; rm -rf $cvsdir
  524. ;;
  525. esac
  526. return $curlret
  527. }
  528. list_dtags() {
  529. {
  530. grep -H '^\[D\] ' package/*/*/*.desc
  531. grep -H '^[X0-9]' target/*/download.txt | sed 's,:,:[D] ,'
  532. } | column_clean
  533. }
  534. list_cksums() {
  535. trap '' INT
  536. # we know we only have single spaces due to list_dtags' column_clean
  537. list_dtags | sed -n \
  538. -e 's,[^ ]* \([X0-9]*\) \(.\)\([^ ]*\) -.*,\1 download/local/\2/\2\3,p' \
  539. -e 's,[^ ]* \([X0-9]*\) \(.\)\([^ ]*\) [^-].*,\1 download/mirror/\2/\2\3,p'
  540. trap INT
  541. }
  542. list() {
  543. trap '' INT
  544. list_cksums | cut -f2- -d' '
  545. trap INT
  546. }
  547. list_unknown() {
  548. trap '' INT
  549. mkdir -p tmp/ ; list | bz2filename > tmp/down.$$.lst
  550. ls download/{Proxy,Proxy-auth,Me,Mirror} \
  551. download/mirror/{README,DOWNTEST,LAST-UPDATE} \
  552. >> tmp/down.$$.lst 2> /dev/null
  553. find download/* -type f -o -type l 2> /dev/null |
  554. while read fn ; do
  555. grep -qx "$fn" tmp/down.$$.lst || echo "Unknown file: $fn"
  556. done
  557. rm -f tmp/down.$$.lst
  558. trap INT
  559. }
  560. list_missing() {
  561. trap '' INT
  562. list | bz2filename | \
  563. while read fn ; do
  564. [ -f "$fn" ] || echo "$fn"
  565. done
  566. trap INT
  567. }
  568. repository() {
  569. for repository ; do
  570. packages `echo package/$repository/*/*.desc`
  571. done
  572. }
  573. required() {
  574. # Choosen config must exist
  575. #
  576. if [ ! -f config/$config/packages ]; then
  577. echo "ERROR: Config $config doesn't exist."
  578. echo "ERROR: try ./scripts/Config -cfg $config first."
  579. exit 1
  580. fi
  581. while read on a b repo pkg c ; do
  582. if [ "$on" = "X" ] ; then
  583. grep -H '^\[D\] ' package/$repo/$pkg/$pkg.desc > tmp/down.$$.lst
  584. while read tag cksum file url ; do
  585. download_file "`source_file cksum $file "$url"`" "$url" "$cksum" "$repo" "$pkg"
  586. done < tmp/down.$$.lst ; rm -f tmp/down.$$.lst
  587. fi
  588. done < config/$config/packages
  589. # NOTE: ROCKCFG -> SDECFG automatized convertion
  590. grep -q 'ROCKCFG' ./config/$config/config &&
  591. sed -i -e 's,ROCKCFG,SDECFG,g' ./config/$config/config
  592. target=`grep '^export SDECFG_TARGET=' config/$config/config | \
  593. cut -f2 -d= | tr -d "'"`
  594. targetchain="$target"; x="$target"
  595. while [ -f "target/$x/extends" ]; do
  596. x="`cat target/$x/extends`"
  597. targetchain="$targetchain $x"
  598. done
  599. for target in $targetchain; do
  600. if [ -f target/$target/download.txt ] ; then
  601. while read cksum file url ; do
  602. download_file "`source_file cksum $file "$url"`" "$url" "$cksum" "$target"
  603. done < target/$target/download.txt
  604. fi
  605. done
  606. }
  607. all() {
  608. local each repo pkg
  609. for repo in $( cd package; ls -1 ); do
  610. for each in package/$repo/*/*.desc; do
  611. pkg="`echo $each | cut -f3 -d/`"
  612. while read tag cksum file url ; do
  613. download_file "`source_file cksum $file "$url"`" "$url" "$cksum" "$repo" "$pkg"
  614. done < <(grep -H '^\[D\] ' package/$repo/$pkg/$pkg.desc)
  615. done
  616. done
  617. for each in target/*/download.txt; do
  618. target="`echo $each | cut -f2 -d/`"
  619. while read cksum file url ; do
  620. download_file "`source_file cksum $file "$url"`" "$url" "$cksum" "$target"
  621. done < <(cat $each)
  622. done
  623. }
  624. package() {
  625. descfile="`echo package/*/$1/$1.desc`"
  626. if [ ! -f $descfile ]; then
  627. echo "Skipping \"$1\" (not found)!"
  628. return
  629. fi
  630. pkg="`echo $descfile | cut -f3 -d/`"
  631. repo="`echo $descfile | cut -f2 -d/`"
  632. while read tag cksum file url ; do
  633. download_file "`source_file cksum $file "$url"`" "$url" "$cksum" "$repo" "$pkg"
  634. done < <(grep -H '^\[D\] ' package/$repo/$pkg/$pkg.desc)
  635. }
  636. packages() {
  637. local descfile
  638. for arg; do
  639. case "$arg" in
  640. target/*)
  641. if [ ! -f $arg ]; then
  642. echo "Skipping \"$arg\" (not found)!"
  643. continue
  644. fi
  645. target="`echo $arg | cut -f2 -d/`"
  646. while read cksum file url ; do
  647. download_file "`source_file cksum $file "$url"`" "$url" "$cksum" "$target"
  648. done < <(cat $arg)
  649. ;;
  650. *)
  651. if [ "${arg%.desc}" != "$arg" ]; then
  652. arg="`echo $arg | cut -f3 -d/`"; fi
  653. # active extensions
  654. local extender=
  655. # pkg_*_{pre,post}.conf is only activated if extender
  656. # is enabled on $config/packages, so we will only
  657. # download files of those extenders
  658. #
  659. for extender in `ls -1 package/*/*/pkg_${arg}_{pre,post}.conf 2> /dev/null |
  660. cut -d/ -f3 | sort -u`; do
  661. if grep -q "^X .* $extender " \
  662. config/$config/packages; then
  663. echo_info "Also downloading $extender ..."
  664. package $extender
  665. fi
  666. done
  667. package $arg
  668. ;;
  669. esac
  670. done
  671. }
  672. set +e
  673. # Things to do only for downloading
  674. #
  675. if [ "${1:0:5}" != "-list" -a $checkonly = 0 ]; then
  676. # we need curl
  677. if [ -z "`type -p curl`" ]; then
  678. echo_abort 2 "we need \`curl\` installed and available on \$PATH to proceed."
  679. fi
  680. # do mirror detection, only once
  681. [ $this_is_the_2nd_run = 1 ] || detect_mirror
  682. fi
  683. case "$1" in
  684. -list) list ;;
  685. -list-unknown) list_unknown ;;
  686. -list-missing) list_missing ;;
  687. -list-cksums) list_cksums ;;
  688. -required) required ;;
  689. -all) all ;;
  690. -repository) shift ; repository "$@" ;;
  691. -*|"") . $SDEROOT/lib/sde-help.in
  692. help_command download
  693. exit 1;;
  694. *) packages "$@" ;;
  695. esac
  696. exit $downloaderr