Beautifulsoup JSON

user_13339694 0
Viewed 54

I am using bs4 to scrape the web. I have a code that gives me the below JSON script tags with info. The aim is to have the "jsonConfig" section as a separate JSON format. I have tried .text() but it returns ''. Would appreciate the help.

from bs4 import BeautifulSoup as soup
from urllib.request import urlopen as uReq
import pandas as pd
import json

my_url= 'https://fashionbunker.com/shop/clothing/dresses?p=1&_=1599036836640'
#open connection and grab the page
uClient=uReq(my_url)
page_html=uClient.read()
#close the connection
uClient.close()

#parse the HTML
page_soup=soup(page_html, "html.parser")

#get all data on the page
container=page_soup.findAll("li",{"class":"item product product-item"})

test=container[0]

p_url=test.div.a["href"]# get new url

#go to the new URL
uClient=uReq(p_url)
page_html=uClient.read()
uClient.close()

#parse the  product HTML
p_page_soup=soup(page_html, "html.parser")

#product name search
p_container=p_page_soup.findAll("div",{"class":"stickyProductName"})
p_test=p_container[0]
p_test.text

#product price search
p_container=p_page_soup.findAll("span",{"class":"price"})
p_test=p_container[0]
p_test.text


#product size search
p_container=p_page_soup.findAll("div",{"class":"product-add-form"})
len(p_container)
p_test=p_container[0]
container2=p_test.find_all("script")[0]
beautifulsoup python
1 Answers

To parse the JSON data from the HTML document, you can use re/json module. For example:

import re
import json
import requests


url = 'https://fashionbunker.com/outbreak-midi-dress-burnt-orange'
html_data = requests.get(url).text

data = re.search(r'let jsonConfig = ({.*})', html_data).group(1)
data = json.loads(data)

# print all data to screen:
print(json.dumps(data, indent=4))

# print some data:
print(data['prices']['finalPrice']['amount'])

Prints:

{
    "attributes": {
        "191": {
            "id": "191",
            "code": "size",
            "label": "Size",
            "options": [
                {
                    "id": "8787",
                    "label": "xxs",
                    "products": [
                        "121995"
                    ]
                },
                {
                    "id": "5487",
                    "label": "xs",
                    "products": [
                        "121992"
                    ]
                },
                {
                    "id": "5485",
                    "label": "s",
                    "products": [
                        "121989"
                    ]
                },
                {
                    "id": "5488",
                    "label": "m",
                    "products": [
                        "121986"
                    ]
                },
                {
                    "id": "5489",
                    "label": "l",
                    "products": [
                        "121983"
                    ]
                },
                {
                    "id": "5522",
                    "label": "xl",
                    "products": [
                        "121980"
                    ]
                }
            ],
            "position": "0"
        }
    },
    "template": "AUD $<%- data.price %>",
    "currencyFormat": "AUD $%s",
    "optionPrices": {
        "121995": {
            "oldPrice": {
                "amount": 229.95
            },
            "basePrice": {
                "amount": 209.04545354545
            },
            "finalPrice": {
                "amount": 229.95
            },
            "tierPrices": [],
            "msrpPrice": {
                "amount": null
            }
        },
        "121992": {
            "oldPrice": {
                "amount": 229.95
            },
            "basePrice": {
                "amount": 209.04545354545
            },
            "finalPrice": {
                "amount": 229.95
            },
            "tierPrices": [],
            "msrpPrice": {
                "amount": null
            }
        },
        "121989": {
            "oldPrice": {
                "amount": 229.95
            },
            "basePrice": {
                "amount": 209.04545354545
            },
            "finalPrice": {
                "amount": 229.95
            },
            "tierPrices": [],
            "msrpPrice": {
                "amount": null
            }
        },
        "121986": {
            "oldPrice": {
                "amount": 229.95
            },
            "basePrice": {
                "amount": 209.04545354545
            },
            "finalPrice": {
                "amount": 229.95
            },
            "tierPrices": [],
            "msrpPrice": {
                "amount": null
            }
        },
        "121983": {
            "oldPrice": {
                "amount": 229.95
            },
            "basePrice": {
                "amount": 209.04545354545
            },
            "finalPrice": {
                "amount": 229.95
            },
            "tierPrices": [],
            "msrpPrice": {
                "amount": null
            }
        },
        "121980": {
            "oldPrice": {
                "amount": 229.95
            },
            "basePrice": {
                "amount": 209.04545354545
            },
            "finalPrice": {
                "amount": 229.95
            },
            "tierPrices": [],
            "msrpPrice": {
                "amount": null
            }
        }
    },
    "priceFormat": {
        "pattern": "AUD $%s",
        "precision": 2,
        "requiredPrecision": 2,
        "decimalSymbol": ".",
        "groupSymbol": ",",
        "groupLength": 3,
        "integerRequired": false
    },
    "prices": {
        "oldPrice": {
            "amount": 229.95
        },
        "basePrice": {
            "amount": 209.04545354545
        },
        "finalPrice": {
            "amount": 229.95
        }
    },
    "productId": "123108",
    "chooseText": "Choose an Option...",
    "images": [],
    "index": {
        "121995": {
            "191": "8787"
        },
        "121992": {
            "191": "5487"
        },
        "121989": {
            "191": "5485"
        },
        "121986": {
            "191": "5488"
        },
        "121983": {
            "191": "5489"
        },
        "121980": {
            "191": "5522"
        }
    },
    "qtys": {
        "121995": 3,
        "121992": 5,
        "121989": 5,
        "121986": 6,
        "121983": 3,
        "121980": 1,
        "hint": "6"
    }
}
229.95
user_100359850
Related