Extract data from a website using Curl and convert it afterwards to C#

user_13718367 0
Viewed 281

I want to read/scap data from gelbeseiten.de, based on the page inspector I Chrome I got a cCURL-statement:

curl "https://www.gelbeseiten.de/AjaxSuche" 
  -H "authority: www.gelbeseiten.de" 
  -H "pragma: no-cache" 
  -H "cache-control: no-cache" 
  -H "sec-ch-ua: \" Not A;Brand\";v=\"99\", \"Chromium\";v=\"98\", \"Google Chrome\";v=\"98\"" 
  -H "sec-ch-ua-mobile: ?0" 
  -H "user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36" 
  -H "sec-ch-ua-platform: \"Windows\"" 
  -H "content-type: multipart/form-data; boundary=----WebKitFormBoundary2Ht3afv9RtrlWwk9" 
  -H "accept: */*" 
  -H "origin: https://www.gelbeseiten.de" 
  -H "sec-fetch-site: same-origin" 
  -H "sec-fetch-mode: cors" 
  -H "sec-fetch-dest: empty" 
  -H "referer: https://www.gelbeseiten.de/" 
  -H "accept-language: de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7" 
  -H "cookie: leverate=eyJjcmVhdGVkIjoxNjQ1MjU1NDE1Mjg5LCJsYXN0QWN0aW9uIjoxNjQ1MjU1NDE1Mjg5LCJyZWZlcnJlciI6IiJ9; leverate_session=1; _wwau=id=2429049831619900000&c=1&ti=1645255415&v=1; _wwas=1645255415; _wwao=; __cmpconsentx15760=CPUoXFgPUoXFgAfbgBDECDCgAAAAAAAAAAigAAAOxgCAA4AYIA3MB2IAAA; __cmpcpcx15760=__1__; __cmpcpc=__1__; __cmpiabli=__56_945_883_772_; __cmpcvcx15760=__s23_c7888_c24201_s24_c7853_c7957_c23722_U__; __cmpcvc=__s23_c7888_c24201_s24_c7853_c7957_c23722_U__; utag_main=v_id:017f10dd56000001b500915ce0d105073003206b00bd0$_sn:1$_ss:0$_st:1645257226251$ses_id:1645255415298%3Bexp-session$_pn:1%3Bexp-session; _wwav=ti=1645255415&pc=2&ec=60&le=1645255445&v=1" 
  --data-raw "------WebKitFormBoundary2Ht3afv9RtrlWwk9

Content-Disposition: form-data; name=\"umkreis\"



-1

------WebKitFormBoundary2Ht3afv9RtrlWwk9

Content-Disposition: form-data; name=\"WAS\"



EDV

------WebKitFormBoundary2Ht3afv9RtrlWwk9

Content-Disposition: form-data; name=\"WO\"



Nürnberg

------WebKitFormBoundary2Ht3afv9RtrlWwk9

Content-Disposition: form-data; name=\"position\"



51

------WebKitFormBoundary2Ht3afv9RtrlWwk9

Content-Disposition: form-data; name=\"anzahl\"



10

------WebKitFormBoundary2Ht3afv9RtrlWwk9

Content-Disposition: form-data; name=\"sortierung\"



relevanz

------WebKitFormBoundary2Ht3afv9RtrlWwk9--

" 
  --compressed

Trying to convert that into C#:

var url = "https://www.gelbeseiten.de/AjaxSuche";

var httpRequest = (HttpWebRequest)WebRequest.Create(url);

httpRequest.Headers["authority"] = "www.gelbeseiten.de";
httpRequest.Headers["pragma"] = "no-cache";
httpRequest.Headers["cache-control"] = "no-cache";
httpRequest.Headers["sec-ch-ua"] = "" Not A;Brand";v="99", "Chromium";v="98", "Google Chrome";v="98"";
httpRequest.Headers["sec-ch-ua-mobile"] = "?0";
httpRequest.Headers["user-agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36";
httpRequest.Headers["sec-ch-ua-platform"] = ""Windows"";
httpRequest.Headers["content-type"] = "multipart/form-data; boundary=----WebKitFormBoundary2Ht3afv9RtrlWwk9";
httpRequest.Headers["accept"] = "*/*";
httpRequest.Headers["origin"] = "https://www.gelbeseiten.de";
httpRequest.Headers["sec-fetch-site"] = "same-origin";
httpRequest.Headers["sec-fetch-mode"] = "cors";
httpRequest.Headers["sec-fetch-dest"] = "empty";
httpRequest.Headers["referer"] = "https://www.gelbeseiten.de/";
httpRequest.Headers["accept-language"] = "de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7";
httpRequest.Headers["cookie"] = "leverate=eyJjcmVhdGVkIjoxNjQ1MjU1NDE1Mjg5LCJsYXN0QWN0aW9uIjoxNjQ1MjU1NDE1Mjg5LCJyZWZlcnJlciI6IiJ9; leverate_session=1; _wwau=id=2429049831619900000&c=1&ti=1645255415&v=1; _wwas=1645255415; _wwao=; __cmpconsentx15760=CPUoXFgPUoXFgAfbgBDECDCgAAAAAAAAAAigAAAOxgCAA4AYIA3MB2IAAA; __cmpcpcx15760=__1__; __cmpcpc=__1__; __cmpiabli=__56_945_883_772_; __cmpcvcx15760=__s23_c7888_c24201_s24_c7853_c7957_c23722_U__; __cmpcvc=__s23_c7888_c24201_s24_c7853_c7957_c23722_U__; utag_main=v_id:017f10dd56000001b500915ce0d105073003206b00bd0$_sn:1$_ss:0$_st:1645257226251$ses_id:1645255415298%3Bexp-session$_pn:1%3Bexp-session; _wwav=ti=1645255415&pc=2&ec=60&le=1645255445&v=1";


var httpResponse = (HttpWebResponse)httpRequest.GetResponse();
using (var streamReader = new StreamReader(httpResponse.GetResponseStream()))
{
   var result = streamReader.ReadToEnd();
}

Console.WriteLine(httpResponse.StatusCode);

Actual Result: There is an Error

Expected result: It should return data

I know I missed some lines - based on the curl-statement -, but I dont know how to convert them into c#.

httpclient web-crawler curl c#
1 Answers

Using https://curl.olsh.me/ you get

var handler = new HttpClientHandler();

// If you are using .NET Core 3.0+ you can replace `~DecompressionMethods.None` to `DecompressionMethods.All`
handler.AutomaticDecompression = ~DecompressionMethods.None; 

// In production code, don't destroy the HttpClient through using, but better reuse an existing instance
// https://www.aspnetmonsters.com/2016/08/2016-08-27-httpclientwrong/
using (var httpClient = new HttpClient(handler))
{
    using (var request = new HttpRequestMessage(new HttpMethod("POST"), "https://www.gelbeseiten.de/AjaxSuche"))
    {
        request.Headers.TryAddWithoutValidation("authority", "www.gelbeseiten.de");
        request.Headers.TryAddWithoutValidation("pragma", "no-cache");
        request.Headers.TryAddWithoutValidation("cache-control", "no-cache");
        request.Headers.TryAddWithoutValidation("sec-ch-ua", "\" Not A;Brand\";v=\"99\", \"Chromium\";v=\"98\", \"Google Chrome\";v=\"98\"");
        request.Headers.TryAddWithoutValidation("sec-ch-ua-mobile", "?0");
        request.Headers.TryAddWithoutValidation("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36");
        request.Headers.TryAddWithoutValidation("sec-ch-ua-platform", "\"Windows\"");
        request.Headers.TryAddWithoutValidation("accept", "*/*");
        request.Headers.TryAddWithoutValidation("origin", "https://www.gelbeseiten.de");
        request.Headers.TryAddWithoutValidation("sec-fetch-site", "same-origin");
        request.Headers.TryAddWithoutValidation("sec-fetch-mode", "cors");
        request.Headers.TryAddWithoutValidation("sec-fetch-dest", "empty");
        request.Headers.TryAddWithoutValidation("referer", "https://www.gelbeseiten.de/");
        request.Headers.TryAddWithoutValidation("accept-language", "de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7");
        request.Headers.TryAddWithoutValidation("cookie", "leverate=eyJjcmVhdGVkIjoxNjQ1MjU1NDE1Mjg5LCJsYXN0QWN0aW9uIjoxNjQ1MjU1NDE1Mjg5LCJyZWZlcnJlciI6IiJ9; leverate_session=1; _wwau=id=2429049831619900000&c=1&ti=1645255415&v=1; _wwas=1645255415; _wwao=; __cmpconsentx15760=CPUoXFgPUoXFgAfbgBDECDCgAAAAAAAAAAigAAAOxgCAA4AYIA3MB2IAAA; __cmpcpcx15760=__1__; __cmpcpc=__1__; __cmpiabli=__56_945_883_772_; __cmpcvcx15760=__s23_c7888_c24201_s24_c7853_c7957_c23722_U__; __cmpcvc=__s23_c7888_c24201_s24_c7853_c7957_c23722_U__; utag_main=v_id:017f10dd56000001b500915ce0d105073003206b00bd0$_sn:1$_ss:0$_st:1645257226251$ses_id:1645255415298%3Bexp-session$_pn:1%3Bexp-session; _wwav=ti=1645255415&pc=2&ec=60&le=1645255445&v=1"); 

        request.Content = new StringContent("------WebKitFormBoundary2Ht3afv9RtrlWwk9\n\nContent-Disposition: form-data; name=\"umkreis\"\n\n\n\n-1\n\n------WebKitFormBoundary2Ht3afv9RtrlWwk9\n\nContent-Disposition: form-data; name=\"WAS\"\n\n\n\nEDV\n\n------WebKitFormBoundary2Ht3afv9RtrlWwk9\n\nContent-Disposition: form-data; name=\"WO\"\n\n\n\nNürnberg\n\n------WebKitFormBoundary2Ht3afv9RtrlWwk9\n\nContent-Disposition: form-data; name=\"position\"\n\n\n\n51\n\n------WebKitFormBoundary2Ht3afv9RtrlWwk9\n\nContent-Disposition: form-data; name=\"anzahl\"\n\n\n\n10\n\n------WebKitFormBoundary2Ht3afv9RtrlWwk9\n\nContent-Disposition: form-data; name=\"sortierung\"\n\n\n\nrelevanz\n\n------WebKitFormBoundary2Ht3afv9RtrlWwk9--\n\n");
        request.Content.Headers.ContentType = MediaTypeHeaderValue.Parse("multipart/form-data; boundary=----WebKitFormBoundary2Ht3afv9RtrlWwk9"); 

        var response = await httpClient.SendAsync(request);
    }
}
user_12581110
Related