I am trying to write a bash script for testing that takes a parameter and sends it through curl to web site. I need to url encode the value to make sure that special characters are processed properly. What is the best way to do this?
Here is my basic script so far:
#!/bin/bash
host=${1:?'bad host'}
value=$2
shift
shift
curl -v -d "param=${value}" http://${host}/somepath $@
Use curl --data-urlencode; from man curl:
This posts data, similar to the other
--dataoptions with the exception that this performs URL-encoding. To be CGI-compliant, the<data>part should begin with a name followed by a separator and a content specification.
Example usage:
curl \
--data-urlencode "paramName=value" \
--data-urlencode "secondParam=value" \
http://example.com
See the man page for more info.
This requires curl 7.18.0 or newer (released January 2008). Use curl -V to check which version you have.
You can as well encode the query string:
curl --get \
--data-urlencode "p1=value 1" \
--data-urlencode "p2=value 2" \
http://example.com
# http://example.com?p1=value%201&p2=value%202
Use Perl's URI::Escape module and uri_escape function in the second line of your bash script:
...
value="$(perl -MURI::Escape -e 'print uri_escape($ARGV[0]);' "$2")"
...
Edit: Fix quoting problems, as suggested by Chris Johnsen in the comments. Thanks!
I find it more readable in python:
encoded_value=$(python3 -c "import urllib.parse; print urllib.parse.quote('''$value''')")
the triple ' ensures that single quotes in value won't hurt. urllib is in the standard library. It work for example for this crazy (real world) url:
"http://www.rai.it/dl/audio/" "1264165523944Ho servito il re d'Inghilterra - Puntata 7
Direct link to awk version : http://www.shelldorado.com/scripts/cmds/urlencode
I used it for years and it works like a charm
:
##########################################################################
# Title : urlencode - encode URL data
# Author : Heiner Steven (heiner.steven@odn.de)
# Date : 2000-03-15
# Requires : awk
# Categories : File Conversion, WWW, CGI
# SCCS-Id. : @(#) urlencode 1.4 06/10/29
##########################################################################
# Description
# Encode data according to
# RFC 1738: "Uniform Resource Locators (URL)" and
# RFC 1866: "Hypertext Markup Language - 2.0" (HTML)
#
# This encoding is used i.e. for the MIME type
# "application/x-www-form-urlencoded"
#
# Notes
# o The default behaviour is not to encode the line endings. This
# may not be what was intended, because the result will be
# multiple lines of output (which cannot be used in an URL or a
# HTTP "POST" request). If the desired output should be one
# line, use the "-l" option.
#
# o The "-l" option assumes, that the end-of-line is denoted by
# the character LF (ASCII 10). This is not true for Windows or
# Mac systems, where the end of a line is denoted by the two
# characters CR LF (ASCII 13 10).
# We use this for symmetry; data processed in the following way:
# cat | urlencode -l | urldecode -l
# should (and will) result in the original data
#
# o Large lines (or binary files) will break many AWK
# implementations. If you get the message
# awk: record `...' too long
# record number xxx
# consider using GNU AWK (gawk).
#
# o urlencode will always terminate it's output with an EOL
# character
#
# Thanks to Stefan Brozinski for pointing out a bug related to non-standard
# locales.
#
# See also
# urldecode
##########################################################################
PN=`basename "$0"` # Program name
VER='1.4'
: ${AWK=awk}
Usage () {
echo >&2 "$PN - encode URL data, $VER
usage: $PN [-l] [file ...]
-l: encode line endings (result will be one line of output)
The default is to encode each input line on its own."
exit 1
}
Msg () {
for MsgLine
do echo "$PN: $MsgLine" >&2
done
}
Fatal () { Msg "$@"; exit 1; }
set -- `getopt hl "$@" 2>/dev/null` || Usage
[ $# -lt 1 ] && Usage # "getopt" detected an error
EncodeEOL=no
while [ $# -gt 0 ]
do
case "$1" in
-l) EncodeEOL=yes;;
--) shift; break;;
-h) Usage;;
-*) Usage;;
*) break;; # First file name
esac
shift
done
LANG=C export LANG
$AWK '
BEGIN {
# We assume an awk implementation that is just plain dumb.
# We will convert an character to its ASCII value with the
# table ord[], and produce two-digit hexadecimal output
# without the printf("%02X") feature.
EOL = "%0A" # "end of line" string (encoded)
split ("1 2 3 4 5 6 7 8 9 A B C D E F", hextab, " ")
hextab [0] = 0
for ( i=1; i<=255; ++i ) ord [ sprintf ("%c", i) "" ] = i + 0
if ("'"$EncodeEOL"'" == "yes") EncodeEOL = 1; else EncodeEOL = 0
}
{
encoded = ""
for ( i=1; i<=length ($0); ++i ) {
c = substr ($0, i, 1)
if ( c ~ /[a-zA-Z0-9.-]/ ) {
encoded = encoded c # safe character
} else if ( c == " " ) {
encoded = encoded "+" # special handling
} else {
# unsafe character, encode it as a two-digit hex-number
lo = ord [c] % 16
hi = int (ord [c] / 16);
encoded = encoded "%" hextab [hi] hextab [lo]
}
}
if ( EncodeEOL ) {
printf ("%s", encoded EOL)
} else {
print encoded
}
}
END {
#if ( EncodeEOL ) print ""
}
' "$@"
url=$(echo "$1" | sed -e 's/%/%25/g' -e 's/ /%20/g' -e 's/!/%21/g' -e 's/"/%22/g' -e 's/#/%23/g' -e 's/\$/%24/g' -e 's/\&/%26/g' -e 's/'\''/%27/g' -e 's/(/%28/g' -e 's/)/%29/g' -e 's/\*/%2a/g' -e 's/+/%2b/g' -e 's/,/%2c/g' -e 's/-/%2d/g' -e 's/\./%2e/g' -e 's/\//%2f/g' -e 's/:/%3a/g' -e 's/;/%3b/g' -e 's//%3e/g' -e 's/?/%3f/g' -e 's/@/%40/g' -e 's/\[/%5b/g' -e 's/\\/%5c/g' -e 's/\]/%5d/g' -e 's/\^/%5e/g' -e 's/_/%5f/g' -e 's/`/%60/g' -e 's/{/%7b/g' -e 's/|/%7c/g' -e 's/}/%7d/g' -e 's/~/%7e/g')
this will encode the string inside of $1 and output it in $url. although you don't have to put it in a var if you want. BTW didn't include the sed for tab thought it would turn it into spaces
If you don't want to depend on Perl you can also use sed. It's a bit messy, as each character has to be escaped individually. Make a file with the following contents and call it urlencode.sed
s/%/%25/g
s/ /%20/g
s/ /%09/g
s/!/%21/g
s/"/%22/g
s/#/%23/g
s/\$/%24/g
s/\&/%26/g
s/'\''/%27/g
s/(/%28/g
s/)/%29/g
s/\*/%2a/g
s/+/%2b/g
s/,/%2c/g
s/-/%2d/g
s/\./%2e/g
s/\//%2f/g
s/:/%3a/g
s/;/%3b/g
s//%3e/g
s/?/%3f/g
s/@/%40/g
s/\[/%5b/g
s/\\/%5c/g
s/\]/%5d/g
s/\^/%5e/g
s/_/%5f/g
s/`/%60/g
s/{/%7b/g
s/|/%7c/g
s/}/%7d/g
s/~/%7e/g
s/ /%09/g
To use it do the following.
STR1=$(echo "https://www.example.com/change&$ ^this to?%checkthe@-functionality" | cut -d\? -f1)
STR2=$(echo "https://www.example.com/change&$ ^this to?%checkthe@-functionality" | cut -d\? -f2)
OUT2=$(echo "$STR2" | sed -f urlencode.sed)
echo "$STR1?$OUT2"
This will split the string into a part that needs encoding, and the part that is fine, encode the part that needs it, then stitches back together.
You can put that into a sh script for convenience, maybe have it take a parameter to encode, put it on your path and then you can just call:
urlencode https://www.exxample.com?isThisFun=HellNo
Python 3 based on @sandro's good answer from 2010:
echo "Test & /me" | python -c "import urllib.parse;print (urllib.parse.quote(input()))"
Test%20%26%20/me
This nodejs-based answer will use encodeURIComponent on stdin:
uriencode_stdin() {
node -p 'encodeURIComponent(require("fs").readFileSync(0))'
}
echo -n $'hello\nwörld' | uriencode_stdin
hello%0Aw%C3%B6rld
In this case, I needed to URL encode the hostname. Don't ask why. Being a minimalist, and a Perl fan, here's what I came up with.
url_encode()
{
echo -n "$1" | perl -pe 's/[^a-zA-Z0-9\/_.~-]/sprintf "%%%02x", ord($&)/ge'
}
Works perfectly for me.
For one of my cases I found that the NodeJS url lib had the simplest solution. Of course YMMV
$ urlencode(){ node -e "console.log(require('url').parse(process.argv.slice(1).join('+')).href)" "$@"; }
$ urlencode "https://example.com?my_database_has=these 'nasty' query strings in it"
https://example.com/?my_database_has=these%20%27nasty%27%20query%20strings%20in%20it
There is an excellent answer from Orwellophile, which does include a pure bash option (function rawurlencode), which I've used on my website (shell based CGI script, large number of URLS in response to search requests). The only draw back was high CPU during peak time.
I've found a modified solution, leverage bash "global replace" feature. With this solution processing time for url encode is 4X faster. The solution identify the characters to be escaped, and uses the "global replace" operator (${var//source/replacement}) to process all substitutions. The speed up is clearly from using bash internal loops, over explicit loop.
Performance: On core i3-8100 3.60Ghz. Test case: 1000 URL from stack overflow, similar to this ticket: "https://stackoverflow.com/questions/296536/how-to-urlencode-data-for-curl-command".
url_encode()
{
local key="${1}" varname="${2:-_rval}" prefix="${3:-_ENCKEY_}"
local unsafe=${key//[-_.~a-zA-Z0-9 ]/}
local -i key_len=${#unsafe}
local ch ch1 ch0
while [ "$unsafe" ] ;do
ch=${unsafe:0:1}
ch0="\\$ch"
printf -v ch1 '%%%02x' "'$ch'"
key=${key//$ch0/"$ch1"}
unsafe=${unsafe//"$ch0"}
done
key=${key// /+}
REPLY="$key"
# printf "%s" "$REPLY"
return 0
}
As a minor extra, it uses '+' to encode the space. Slightly more compact URL.
Benchmark:
function t {
local key
for (( i=1 ; i<=$1 ; i++ )) do url_encode "$2" kkk2 ; done
echo "K=$REPLY"
}
t 1000 "https://stackoverflow.com/questions/296536/how-to-urlencode-data-for-curl-command"
Note
#!/bin/dash
replaceUnicodes () { # $1=input/output file
if ! mv -f "$1" "$1".tmp 2>/dev/null; then return 1; fi
output="$1" awk '
function hexValue(chr) {
if(chr=="0") return 0; if(chr=="1") return 1; if(chr=="2") return 2; if(chr=="3") return 3; if(chr=="4") return 4; if(chr=="5") return 5;
if(chr=="6") return 6; if(chr=="7") return 7; if(chr=="8") return 8; if(chr=="9") return 9; if(chr=="A") return 10;
if(chr=="B") return 11; if(chr=="C") return 12; if(chr=="D") return 13; if(chr=="E") return 14; return 15 }
function hexToDecimal(str, value,i,inc) {
str=toupper(str); value=and(hexValue(substr(str,length(str),1)),15); inc=1;
for(i=length(str)-1;i>0;i--) {
value+=lshift(hexValue(substr(str,i,1)),4*inc++)
} return value }
function toDecimal(str, value,i) {
for(i=1;i<=length(str);i++) {
value=(value*10)+substr(str,i,1)
} return value }
function to32BE(high,low) {
# return 0x10000+((high-0xD800)*0x400)+(low-0xDC00) }
return lshift((high-0xD800),10)+(low-0xDC00)+0x10000 }
function toUTF8(value) {
if(value<0x80) {
return sprintf("%%%02X",value)
} else if(value>0xFFFF) {
return sprintf("%%%02X%%%02X%%%02X%%%02X",or(0xF0,and(rshift(value,18),0x07)),or(0x80,and(rshift(value,12),0x3F)),or(0x80,and(rshift(value,6),0x3F)),or(0x80,and(rshift(value,0),0x3F)))
} else if(value>0x07FF) {
return sprintf("%%%02X%%%02X%%%02X",or(0xE0,and(rshift(value,12),0x0F)),or(0x80,and(rshift(value,6),0x3F)),or(0x80,and(rshift(value,0),0x3F)))
} else { return sprintf("%%%02X%%%02X",or(0xC0,and(rshift(value,6),0x1F)),or(0x80,and(rshift(value,0),0x3F))) }
}
function trap(str) { sub(/^\\+/,"\\",str); return str }
function esc(str) { gsub(/\\/,"\\\\",str); return str }
BEGIN { output=ENVIRON["output"] }
{
finalStr=""; while(match($0,/[\\]+u[0-9a-fA-F]{4}/)) {
p=substr($0,RSTART,RLENGTH); num=hexToDecimal(substr(p,RLENGTH-3,4));
bfrStr=substr($0,1,RSTART-1); $0=substr($0,RSTART+RLENGTH,length($0)-(RSTART+RLENGTH-1));
if(surrogate) {
surrogate=0;
if(RSTART!=1 || num<0xD800 || (num>0xDBFF && num<0xDC00) || num>0xDFFF) {
finalStr=sprintf("%s%s%s%s",finalStr,trap(highP),bfrStr,toUTF8(num))
} else if(num>0xD7FF && num<0xDC00) {
surrogate=1; high=num; finalStr=sprintf("%s%s",finalStr,trap(highP))
} else { finalStr=sprintf("%s%s",finalStr,toUTF8(to32BE(high,num))) }
} else if(num>0xD7FF && num<0xDC00) {
surrogate=1; highP=p; high=num; finalStr=sprintf("%s%s",finalStr,bfrStr)
} else { finalStr=sprintf("%s%s%s",finalStr,bfrStr,toUTF8(num)) }
} finalStr=sprintf("%s%s",finalStr,$0); $0=finalStr
while(match($0,/[\\]+U[0-9a-fA-F]{8}/)) {
str=substr($0,RSTART,RLENGTH); gsub(esc(str),toUTF8(hexToDecimal(substr(str,RLENGTH-7,8))),$0)
}
while(match($0,/[\\]*&#[xX][0-9a-fA-F]{1,8};/)) {
str=substr($0,RSTART,RLENGTH); idx=index(str,"#");
gsub(esc(str),toUTF8(hexToDecimal(substr(str,idx+2,RLENGTH-idx-2))),$0)
}
while(match($0,/[\\]*&#[0-9]{1,10};/)) {
str=substr($0,RSTART,RLENGTH); idx=index(str,"#");
gsub(esc(str),toUTF8(toDecimal(substr(str,idx+1,RLENGTH-idx-1))),$0)
}
printf("%s\n",$0) > output
}' "$1".tmp
rm -f "$1".tmp
}
replaceHtmlEntities () { # $1=input/output file
if ! mv -f "$1" "$1".tmp 2>/dev/null; then return 1; fi
sed 's/%3[aA]/:/g; s/%2[fF]/\//g; s/"/%22/g; s/</%3C/g; s/>/%3E/g; s/ /%A0/g; s/¢/%A2/g; s/£/%A3/g; s/¥/%A5/g; s/©/%A9/g; s/®/%AE/g; s/&/\&/g; s/\\*\//\//g' "$1".tmp > "$1"
rm -f "$1".tmp
}
# "od -v -A n -t u1 -w99999999"
# "hexdump -v -e \47/1 \42%d \42\47"
# Reminder :: Do not encode (, ), [, and ].
toUTF8Encoded () { # $1=input/output file
if ! mv -f "$1" "$1".tmp 2>/dev/null; then return 1; fi
if [ -s "$1".tmp ]; then
# od -A n -t u1 -w99999999 "$1".tmp | \
hexdump -v -e '/1 "%d "' "$1".tmp | \
output="$1" awk 'function hexDigit(chr) { if((chr>47 && chr<58) || (chr>64 && chr<71) || (chr>96 && chr<103)) return 1; return 0 }
BEGIN { output=ENVIRON["output"] }
{ for(i=1;i<=NF;i++) {
flushed=0; c=$(i);
if(c==13) { if($(i+1)==10) i++; printf("%s\n",url) > output; url=""; flushed=1
} else if(c==10) { printf("%s\n",url) > output; url=""; flushed=1
} else if(c==37) {
if(hexDigit($(i+1)) && hexDigit($(i+2))) {
url=sprintf("%s%%%c%c",url,$(i+1),$(i+2)); i+=2
} else { url=sprintf("%s%%25",url) }
} else if(c>32 && c<127 && c!=34 && c!=39 && c!=96 && c!=60 && c!=62) {
url=sprintf("%s%c",url,c)
} else { url=sprintf("%s%%%02X",url,c) }
} if(!flushed) printf("%s\n",url) > output
}'
fi
rm -f "$1".tmp
}
Call replaceUnicodes() --> replaceHtmlEntities() --> toUTF8Encoded()