Scrapy Request body dict to byte string

Question

I have a scrapy start_request function where I want to send request with body. When I copy it from curl I get body in string which I can pass as body.

sending it like yield scrapy.http.Request(url=url, headers=headers, body=body, method="POST") works fine.

However for further adjustments I want to have it as dict data so I can change some variables.

when I send it as yield scrapy.http.Request(url=url, headers=headers, body=json.dumps(data), method="POST") it does not work. I tried encode but no success. Is problem in sending the request or in parse function? Whole code below.

def start_requests(self):
    url = 'https://www.tradeinn.com/index.php?action=get_info_elastic_listado&id_tienda=10&idioma=eng'

    headers = {
    "authority": "www.tradeinn.com",
    "accept": "application/json, text/javascript, */*; q=0.01",
    "accept-language": "en-GB,en-US;q=0.9,en;q=0.8",
    "content-type": "application/x-www-form-urlencoded; charset=UTF-8",
    "origin": "https://www.tradeinn.com",
    "referer": "https://www.tradeinn.com/runnerinn/en/mens-shoes-trail-running-shoes/10005/s",
    "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36",
    "x-requested-with": "XMLHttpRequest"
}

   body = 'vars%5B%5D=id_familia%3D10002&vars%5B%5D=atributos_e%3D5091%2C6017&vars%5B%5D=model.eng%3Bmodel.eng%3Bvideo_mp4%3Bid_marca%3Bprecio_tachado%3Bsostenible%3Bproductes.talla2%3Bproductes.talla_usa%3Bproductes.talla_jp%3Bproductes.talla_uk%3Btres_sesenta%3Batributos_padre.atributos.id_atribut_valor%3Bproductes.v360%3Bproductes.v180%3Bproductes.v90%3Bproductes.v30%3Bproductes.exist%3Bproductes.stock_reservat%3Bproductes.pmp%3Bproductes.id_producte%3Bproductes.color%3Bproductes.referencia%3Bproductes.brut%3Bproductes.desc_brand%3Bimage_created%3Bid_modelo%3Bfamilias.eng%3Bfamilias.eng%3Bfamilias.id_familia%3Bfamilias.subfamilias.eng%3Bfamilias.subfamilias.eng%3Bfamilias.subfamilias.id_tienda%3Bfamilias.subfamilias.id_subfamilia%3Bproductes.talla%3Bproductes.baja%3Bproductes.rec%3Bprecio_win_209%3Bproductes.sellers.id_seller%3Bproductes.sellers.precios_paises.precio%3Bproductes.sellers.precios_paises.id_pais%3Bfecha_descatalogado%3Bmarca%3Bproductes.talla_uk&vars%5B%5D=v30_sum%3Bdesc%40tm10%3Basc&vars%5B%5D=48&vars%5B%5D=productos&vars%5B%5D=search&vars%5B%5D=id_subfamilia%3D10005&vars%5B%5D=96&texto_search='
   data = {
    'vars[]': [
        'id_familia=10002',
        'atributos_e=5091,6017',
        'model.eng;model.eng;video_mp4;id_marca;precio_tachado;sostenible;productes.talla2;productes.talla_usa;productes.talla_jp;productes.talla_uk;tres_sesenta;atributos_padre.atributos.id_atribut_valor;productes.v360;productes.v180;productes.v90;productes.v30;productes.exist;productes.stock_reservat;productes.pmp;productes.id_producte;productes.color;productes.referencia;productes.brut;productes.desc_brand;image_created;id_modelo;familias.eng;familias.eng;familias.id_familia;familias.subfamilias.eng;familias.subfamilias.eng;familias.subfamilias.id_tienda;familias.subfamilias.id_subfamilia;productes.talla;productes.baja;productes.rec;precio_win_209;productes.sellers.id_seller;productes.sellers.precios_paises.precio;productes.sellers.precios_paises.id_pais;fecha_descatalogado;marca;productes.talla_uk',
        'v30_sum;desc@tm10;asc',
        '48',
        'productos',
        'search',
        'id_subfamilia=10005',
        '96',
    ],
    'texto_search': '',
}

    yield scrapy.http.Request(url=url, headers=headers, body=body, method="POST")

def parse(self, response):
    data = json.loads(response.body)
    yield from data["id_modelos"]

userphOc1 · Answer

The reason that you do not receive the same response using json that you do with the percent-encoded string is because that is what the server accepts and knows how to translate.

Converting percent-encoded strings to a dictionary and then back again can be easily achieved using the python's urllib.parse module.

for example:

from urllib.parse import urlencode, parse_qs

body = 'vars%5B%5D=id_familia%3D10002&vars%5B%5D=atributos_e%3D5091%2C6017&vars%5B%5D=model.eng%3Bmodel.eng%3Bvideo_mp4%3Bid_marca%3Bprecio_tachado%3Bsostenible%3Bproductes.talla2%3Bproductes.talla_usa%3Bproductes.talla_jp%3Bproductes.talla_uk%3Btres_sesenta%3Batributos_padre.atributos.id_atribut_valor%3Bproductes.v360%3Bproductes.v180%3Bproductes.v90%3Bproductes.v30%3Bproductes.exist%3Bproductes.stock_reservat%3Bproductes.pmp%3Bproductes.id_producte%3Bproductes.color%3Bproductes.referencia%3Bproductes.brut%3Bproductes.desc_brand%3Bimage_created%3Bid_modelo%3Bfamilias.eng%3Bfamilias.eng%3Bfamilias.id_familia%3Bfamilias.subfamilias.eng%3Bfamilias.subfamilias.eng%3Bfamilias.subfamilias.id_tienda%3Bfamilias.subfamilias.id_subfamilia%3Bproductes.talla%3Bproductes.baja%3Bproductes.rec%3Bprecio_win_209%3Bproductes.sellers.id_seller%3Bproductes.sellers.precios_paises.precio%3Bproductes.sellers.precios_paises.id_pais%3Bfecha_descatalogado%3Bmarca%3Bproductes.talla_uk&vars%5B%5D=v30_sum%3Bdesc%40tm10%3Basc&vars%5B%5D=48&vars%5B%5D=productos&vars%5B%5D=search&vars%5B%5D=id_subfamilia%3D10005&vars%5B%5D=96&texto_search='
data = parse_qs(body)  # <-- this gives you a dict
print(data, "

") 

# make some changes to the dictionary

body = urlencode(data, doseq=True)  # <--- converts to percent-encoded string
print(body)

output:

{'vars[]': ['id_familia=10002', 'atributos_e=5091,6017', 'model.eng;model.eng;video_mp4;id_marca;precio_tachado;sostenible;productes.talla2;productes.talla_usa;productes.talla_jp;productes.talla_uk;tres_s
esenta;atributos_padre.atributos.id_atribut_valor;productes.v360;productes.v180;productes.v90;productes.v30;productes.exist;productes.stock_reservat;productes.pmp;productes.id_producte;productes.color;pro
ductes.referencia;productes.brut;productes.desc_brand;image_created;id_modelo;familias.eng;familias.eng;familias.id_familia;familias.subfamilias.eng;familias.subfamilias.eng;familias.subfamilias.id_tienda
;familias.subfamilias.id_subfamilia;productes.talla;productes.baja;productes.rec;precio_win_209;productes.sellers.id_seller;productes.sellers.precios_paises.precio;productes.sellers.precios_paises.id_pais
;fecha_descatalogado;marca;productes.talla_uk', 'v30_sum;desc@tm10;asc', '48', 'productos', 'search', 'id_subfamilia=10005', '96']}


vars%5B%5D=id_familia%3D10002&vars%5B%5D=atributos_e%3D5091%2C6017&vars%5B%5D=model.eng%3Bmodel.eng%3Bvideo_mp4%3Bid_marca%3Bprecio_tachado%3Bsostenible%3Bproductes.talla2%3Bproductes.talla_usa%3Bproducte
s.talla_jp%3Bproductes.talla_uk%3Btres_sesenta%3Batributos_padre.atributos.id_atribut_valor%3Bproductes.v360%3Bproductes.v180%3Bproductes.v90%3Bproductes.v30%3Bproductes.exist%3Bproductes.stock_reservat%3
Bproductes.pmp%3Bproductes.id_producte%3Bproductes.color%3Bproductes.referencia%3Bproductes.brut%3Bproductes.desc_brand%3Bimage_created%3Bid_modelo%3Bfamilias.eng%3Bfamilias.eng%3Bfamilias.id_familia%3Bfa
milias.subfamilias.eng%3Bfamilias.subfamilias.eng%3Bfamilias.subfamilias.id_tienda%3Bfamilias.subfamilias.id_subfamilia%3Bproductes.talla%3Bproductes.baja%3Bproductes.rec%3Bprecio_win_209%3Bproductes.sell
ers.id_seller%3Bproductes.sellers.precios_paises.precio%3Bproductes.sellers.precios_paises.id_pais%3Bfecha_descatalogado%3Bmarca%3Bproductes.talla_uk&vars%5B%5D=v30_sum%3Bdesc%40tm10%3Basc&vars%5B%5D=48&v
ars%5B%5D=productos&vars%5B%5D=search&vars%5B%5D=id_subfamilia%3D10005&vars%5B%5D=96

For your example, once you have made the desired changes to the data dict, you can simply convert the dict back into the correct format and submit it as the parameter to the request object.

for example:

from urllib.parse import urlencode
...
...

yield scrapy.http.Request(url=url, headers=headers, body=urlencode(data, doseq=True), method="POST")

Scrapy Request body dict to byte string

1 Answers