import time
from tqdm import tqdm
from requests import Session
from bs4 import BeautifulSoup
from headers import headers
from proxy_randomizer import RegisteredProviders
s = Session()
rp = RegisteredProviders()
rp.parse_providers()
prox = rp.get_random_proxy()
# print(prox)
urls = []
for i in range(1, 10):
r = s.get('https://www.newegg.com/Power-Supplies/SubCategory/ID-58?Tid=7657',
params={'page': i}, headers=headers, proxies=prox)
print(prox)
soup = BeautifulSoup(r.text, 'lxml')
boxes = soup.find_all('div', {'class': 'item-cell'})
time.sleep(3)
for l in boxes:
links = l.find('a', {'class': 'item-img'}).get('href')
for link in tqdm(links):
urls.append(links)
with open('useful_links.txt', 'a') as f:
for line in urls:
f.write(f'{line}\n')
error:
Traceback (most recent call last):
File "/home/anatolii/PycharmProjects/pythonProject/newegg.py", line 17, in <module>
r = s.get('https://www.newegg.com/Power-Supplies/SubCategory/ID-58?Tid=7657',
File "/home/anatolii/PycharmProjects/pythonProject/lib/python3.10/site-packages/requests/sessions.py", line 600, in get
return self.request("GET", url, **kwargs)
File "/home/anatolii/PycharmProjects/pythonProject/lib/python3.10/site-packages/requests/sessions.py", line 577, in request
settings = self.merge_environment_settings(
File "/home/anatolii/PycharmProjects/pythonProject/lib/python3.10/site-packages/requests/sessions.py", line 758, in merge_environment_settings
no_proxy = proxies.get("no_proxy") if proxies is not None else None
AttributeError: 'Proxy' object has no attribute 'get'