I am trying to use BeautifulSoup to retrieve the text of a certain element within markup strings that are stored throughout one column of a pd.DataFrame.
pd.DataFrame.info()
<class 'pandas.core.frame.DataFrame'>
Index: 55778 entries, 0 to html
Data columns (total 1 columns):
# Column Non-Null Count Dtype
--- ------ -------------- -----
0 html 55778 non-null object
dtypes: object(1)
memory usage: 2.9+ MB
Code
import bs4
import pandas as pd
def get_location_text(markup: str) -> str:
selector = ".column-sm-2" # "div > ul > li"
location = bs4.BeautifulSoup(markup, "lxml").select_one(selector)
location_text = location.get_text(strip=True)
return location_text
df_locations["text"] = df_locations["html"].apply(get_location_text)
Error Message
---------------------------------------------------------------------------
AttributeError Traceback (most recent call last)
/var/folders/bl/b9g4bbkj381ghzzvws3tmcgm0000gn/T/ipykernel_50020/1441714440.py in <module>
----> 1 df_locations["text"] = df_locations["html"].apply(get_location_text)
~/bert2bert/venv/lib/python3.7/site-packages/pandas/core/series.py in apply(self, func, convert_dtype, args, **kwargs)
4355 dtype: float64
4356 """
-> 4357 return SeriesApply(self, func, convert_dtype, args, kwargs).apply()
4358
4359 def _reduce(
~/bert2bert/venv/lib/python3.7/site-packages/pandas/core/apply.py in apply(self)
1041 return self.apply_str()
1042
-> 1043 return self.apply_standard()
1044
1045 def agg(self):
~/bert2bert/venv/lib/python3.7/site-packages/pandas/core/apply.py in apply_standard(self)
1099 values,
1100 f, # type: ignore[arg-type]
-> 1101 convert=self.convert_dtype,
1102 )
1103
~/bert2bert/venv/lib/python3.7/site-packages/pandas/_libs/lib.pyx in pandas._libs.lib.map_infer()
/var/folders/bl/b9g4bbkj381ghzzvws3tmcgm0000gn/T/ipykernel_50020/222721594.py in get_location_text(markup)
1 def get_location_text(markup: str) -> str:
2 selector = ".colcount-sm-2" # "div > ul > li"
----> 3 location = bs4.BeautifulSoup(markup, "lxml").select_one(selector)
4 location_text = location.get_text(strip=True)
5 return location_text
~/bert2bert/venv/lib/python3.7/site-packages/bs4/__init__.py in __init__(self, markup, features, builder, parse_only, from_encoding, exclude_encodings, element_classes, **kwargs)
327 self.contains_replacement_characters) in (
328 self.builder.prepare_markup(
--> 329 markup, from_encoding, exclude_encodings=exclude_encodings)):
330 self.reset()
331 self.builder.initialize_soup(self)
~/bert2bert/venv/lib/python3.7/site-packages/bs4/builder/_lxml.py in prepare_markup(self, markup, user_specified_encoding, exclude_encodings, document_declared_encoding)
180 # We're in HTML mode, so if we're given XML, that's worth
181 # noting.
--> 182 DetectsXMLParsedAsHTML.warn_if_markup_looks_like_xml(markup)
183 else:
184 self.processing_instruction_class = XMLProcessingInstruction
~/bert2bert/venv/lib/python3.7/site-packages/bs4/builder/__init__.py in warn_if_markup_looks_like_xml(cls, markup)
533
534 if (markup is not None
--> 535 and markup.startswith(prefix)
536 and not looks_like_html.search(markup[:500])
537 ):
~/bert2bert/venv/lib/python3.7/site-packages/pandas/core/generic.py in __getattr__(self, name)
5485 ):
5486 return self[name]
-> 5487 return object.__getattribute__(self, name)
5488
5489 def __setattr__(self, name: str, value) -> None:
AttributeError: 'Series' object has no attribute 'startswith'
In the comment section requested debugging output
df_locations["html"].describe()
count 55778
unique 17077
top <section class="mfp-hide modal-dialog modal-co...
freq 3709
Name: html, dtype: object
df_locations["html"].head(2)
0 <section class="mfp-hide modal-dialog modal-co...
4 <section class="mfp-hide modal-dialog modal-co...
Name: html, dtype: object
type(df_locations["html"].iloc[0])
<class 'str'>