I was trying to encode in scraping part and decode in cleaning but I was having same code or HTTP 400 code, I run code on Spyder 3.7, also tried Jupyter 3 Python. I rea a lot about it online, was trying apply
So my error:
runfile('C:/Users/sound/Desktop/dataPlot.py', wdir='C:/Users/sound/Desktop')
Traceback (most recent call last):
File "<ipython-input-248-b48d632496ca>", line 1, in <module>
runfile('C:/Users/sound/Desktop/dataPlot.py', wdir='C:/Users/sound/Desktop')
File "C:\Users\sound\Anaconda3\lib\site-packages\spyder_kernels\customize\spydercustomize.py", line 786, in runfile
execfile(filename, namespace)
File "C:\Users\sound\Anaconda3\lib\site-packages\spyder_kernels\customize\spydercustomize.py", line 110, in execfile
exec(compile(f.read(), filename, 'exec'), namespace)
File "C:/Users/sound/Desktop/dataPlot.py", line 16, in <module>
jobs = sns.load_dataset(needToPlot)
File "C:\Users\sound\Anaconda3\lib\site-packages\seaborn\utils.py", line 428, in load_dataset
urlretrieve(full_path, cache_path)
File "C:\Users\sound\Anaconda3\lib\urllib\request.py", line 247, in urlretrieve
with contextlib.closing(urlopen(url, data)) as fp:
File "C:\Users\sound\Anaconda3\lib\urllib\request.py", line 222, in urlopen
return opener.open(url, data, timeout)
File "C:\Users\sound\Anaconda3\lib\urllib\request.py", line 525, in open
response = self._open(req, data)
File "C:\Users\sound\Anaconda3\lib\urllib\request.py", line 543, in _open
'_open', req)
File "C:\Users\sound\Anaconda3\lib\urllib\request.py", line 503, in _call_chain
result = func(*args)
File "C:\Users\sound\Anaconda3\lib\urllib\request.py", line 1360, in https_open
context=self._context, check_hostname=self._check_hostname)
File "C:\Users\sound\Anaconda3\lib\urllib\request.py", line 1317, in do_open
encode_chunked=req.has_header('Transfer-encoding'))
File "C:\Users\sound\Anaconda3\lib\http\client.py", line 1229, in request
self._send_request(method, url, body, headers, encode_chunked)
File "C:\Users\sound\Anaconda3\lib\http\client.py", line 1240, in _send_request
self.putrequest(method, url, **skips)
File "C:\Users\sound\Anaconda3\lib\http\client.py", line 1107, in putrequest
self._output(request.encode('ascii'))
UnicodeEncodeError: 'ascii' codec can't encode character '\u0117' in position 1760: ordinal not in range(128)
And my code: I've started with scraping part to gather data about IT positions and Salary from popular jobspage.
#Scraping part
import requests
import pandas as pd
import seaborn as sns
from bs4 import BeautifulSoup
url = 'https://www.cvbankas.lt/?padalinys%5B0%5D=76&page=1'
soup = BeautifulSoup(requests.get(url).content, 'html.parser')
all_data = []
for i in range(1, 9):
url = 'https://www.cvbankas.lt/?padalinys%5B0%5D=76&page='+str(i)
print(url)
soup = BeautifulSoup(requests.get(url).content, 'html.parser')
for h3 in soup.select('h3.list_h3'):
try:
job_title = h3.get_text(strip=True)
company = h3.find_next(class_="heading_secondary").get_text(strip=True)
salary = h3.find_next(class_="salary_amount").get_text(strip=True)
location = h3.find_next(class_="list_city").get_text(strip=True)
print('{:<50} {:<15} {:<15} {}'.format(company, salary, location, job_title))
except AttributeError:
pass
all_data.append({
'Job Title': job_title,
'Company': company,
'Salary': salary,
'Location': location
})
df = pd.DataFrame(all_data)
df.to_csv('data.csv')
After I saved four types of text into dictionary. I tried to clean it so it would be easier to plot it out
#Cleaning part
needtoclean = pd.read_csv(r'C:\Users\sound\Desktop\data.csv')
#making all into str, due its mixed with int,float,str
needtoclean['Salary'] = needtoclean['Salary'].astype(str)
cleanedSalary = []
columnSeriesObj = needtoclean['Salary']
for value in columnSeriesObj.values:
value = value.replace('Nuo',"").replace('Iki','')
value = value.strip()
value = re.split(r'[\s,-]+', value)
if len(value) > 1: #
value = (int(value[0])+int(value[1]))/2
else:
value = float(value[0])
value = round(value)
cleanedSalary.append(value)
needtoclean['Salary'] = cleanedSalary
cleanTitles = []
for value in needtoclean['Job Title'].values:
value = value.title()
value = value.replace('/', ' ').replace('(-Ė)','').replace('(-A)','').replace('(-As)', '').replace('(0s)','')
value = value.strip()
value = value.split()
cleanTitles.append(value)
needtoclean['Job Title'] = cleanTitles
needtoclean.to_csv(r'C:/Users/sound/Desktop/manodata.csv')
After cleaning I cannot start plotting due I getting ascii error, not sure which approach should be fine for decoding.
#Ploting part
needToPlot = pd.read_csv(r'C:\Users\sound\Desktop\manodata.csv')
jobs = sns.load_dataset(needToPlot)