diff --git a/lib/sde-download/get-tokenized.sh b/lib/sde-download/get-tokenized.sh new file mode 100755 index 0000000..3ed7219 --- /dev/null +++ b/lib/sde-download/get-tokenized.sh @@ -0,0 +1,40 @@ +#!/bin/sh +# --- SDE-COPYRIGHT-NOTE-BEGIN --- +# This copyright note is auto-generated by ./scripts/Create-CopyPatch. +# +# Filename: lib/sde-download/get-tokenized.sh +# Copyright (C) 2009 The OpenSDE Project +# +# More information can be found in the files COPYING and README. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. A copy of the +# GNU General Public License can be found in the file COPYING. +# --- SDE-COPYRIGHT-NOTE-END --- + +# returns mutilated but grep friendly content of a page +# +# (It will include many empty lines, but it doesn't worth to remove them +# here) +# + +curl_options="-s" +curl_options="$curl_options --disable-epsv" +curl_options="$curl_options -y 60 -Y 1 --connect-timeout 300" + +for x; do + protocol="${x%%://*}" + case "$protocol" in + http|https|ftp) + ;; + *) + echo "$protocol: not supported" >&2 + continue + ;; + esac + echo "$x" >&2 + curl $curl_options "$x" + errno="$?" +done < /dev/null | sed -e "s,[\r\t\"'<>= /],\n,g" +exit "${errno:-1}"