sde-download: Introduced get-tokenized.sh to get the tokenized output of an URL

lib/sde-download/get-tokenized.sh

@@ -0,0 +1,40 @@
+#!/bin/sh
+# --- SDE-COPYRIGHT-NOTE-BEGIN ---
+# This copyright note is auto-generated by ./scripts/Create-CopyPatch.
+#
+# Filename: lib/sde-download/get-tokenized.sh
+# Copyright (C) 2009 The OpenSDE Project
+#
+# More information can be found in the files COPYING and README.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License. A copy of the
+# GNU General Public License can be found in the file COPYING.
+# --- SDE-COPYRIGHT-NOTE-END ---
+
+# returns mutilated but grep friendly content of a page
+#
+# (It will include many empty lines, but it doesn't worth to remove them
+# here)
+#
+
+curl_options="-s"
+curl_options="$curl_options --disable-epsv"
+curl_options="$curl_options -y 60 -Y 1 --connect-timeout 300"
+
+for x; do
+	protocol="${x%%://*}"
+	case "$protocol" in
+	http|https|ftp)
+		;;
+	*)
+		echo "$protocol: not supported" >&2
+		continue
+		;;
+	esac
+	echo "$x" >&2
+	curl $curl_options "$x"
+	errno="$?"
+done < /dev/null | sed -e "s,[\r\t\"'<>= /],\n,g"
+exit "${errno:-1}"