How can I filter website request by price?

Viewed 53

I need to filter a request from a website by bins of price. Since the site only allows for the maximum of 10000 cases I need to filter my request.

For instance: Extract all that, in a first round, have prices between 0:100000, then, in a second round 100001 : 200000 etc... until the maximum value.

But for that where can I make that filter and how can I find the maximum value of all?

import requests
import math
import pandas as pd

url = 'https://www.remax.pt/Api/Listing/MultiMatchSearch'
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36'}
payload = {"filters":
           [{"field":"BusinessTypeID", "value":"1", "type":0},
            {"field":"ListingClassID", "value":"1", "type":0}],
           "sort":{"fieldToSort":"ContractDate", "order":1}}
size = 10000
query_payload = {
'page':0,
'searchValue':'',
'size':size}

    
jsonData = requests.post(url, params=query_payload,json=payload).json()
total_pages =  math.ceil(jsonData['total'] / size) 

element_list = []
for page in range(0, total_pages + 1):
    if page != 0:
        query_payload.update({'page':page})
        jsonData = requests.post(url, params=query_payload,json=payload, headers=headers).json()
        
    if len(jsonData['results']) == 0:
        break
    element_list += jsonData['results']
    print('Collected: Page %s: %s listings' %(page+1, len(jsonData['results'])))
    
listing_df = pd.DataFrame(element_list)    

rows = []
listing_ids = list(listing_df['listingTitle'])
for idx, listing_id in enumerate(listing_ids):
    listing_url = 'https://s.maxwork.pt/site/static/9/listings/details/%s.html' %listing_id
    response = requests.get(listing_url, headers=headers)
    temp_details = pd.read_html(response.text)[0]
    
    alpha = temp_details.iloc[:,[0, 1]]
    beta = temp_details.iloc[:,[2, 3]]
    beta.columns = alpha.columns
    details = pd.concat([alpha,beta])
    details['listingTitle'] = listing_id
    details = details.pivot(index='listingTitle', columns=0, values=1).reset_index(drop=False)
    rows += details.to_dict(orient='records')

    print('%s of %s' %(idx+1, len(listing_ids)))

listing_desc_df = pd.DataFrame(rows)

df = listing_df.merge(listing_desc_df, how='left', on=['listingTitle'])
0 Answers
Related