I have around 30000 Urls in my csv. I need to check if it has meta content is present or not, for each url. I am using request_cache to basically cache the response to a sqlite db. It was taking about 24hrs even after using a caching sys. Therefore I moved to concurrency. I think I have done something wrong with out = executor.map(download_site, sites, headers). And do not know how to fix it.
AttributeError: 'str' object has no attribute 'items'
import concurrent.futures
import requests
import threading
import time
import pandas as pd
import requests_cache
from PIL import Image
from io import BytesIO
thread_local = threading.local()
df = pd.read_csv("test.csv")
sites = []
for row in df['URLS']:
sites.append(row)
# print("URL is shortened")
user_agent = 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.7) Gecko/2009021910 Firefox/3.0.7'
headers={'User-Agent':user_agent,}
requests_cache.install_cache('network_call', backend='sqlite', expire_after=2592000)
def getSess():
if not hasattr(thread_local, "session"):
thread_local.session = requests.Session()
return thread_local.session
def networkCall(url, headers):
print("In Download site")
session = getSess()
with session.get(url, headers=headers) as response:
print(f"Read {len(response.content)} from {url}")
return response.content
out = []
def getMeta(meta_res):
print("Get data")
for each in meta_res:
meta = each.find_all('meta')
for tag in meta:
if 'name' in tag.attrs.keys() and tag.attrs['name'].strip().lower() in ['description', 'keywords']:
content = tag.attrs['content']
if content != '':
out.append("Absent")
else:
out.append("Present")
return out
def allSites(sites):
with concurrent.futures.ThreadPoolExecutor(max_workers=50) as executor:
out = executor.map(networkCall, sites, headers)
return list(out)
if __name__ == "__main__":
sites = [
"https://www.jython.org",
"http://olympus.realpython.org/dice",
] * 15000
start_time = time.time()
list_meta = allSites(sites)
print("META ", list_meta)
duration = time.time() - start_time
print(f"Downloaded {len(sites)} in {duration} seconds")
output = getMeta(list_meta)
df["is it there"] = pd.Series(output)
df.to_csv('new.csv',index=False, header=True)