I wrote a script that scrolls through an infinitely loading table on a site and scrapes the entries, but instead of being a the entire thing is made up of elements. I can't scroll through and then scrape since new elements are loaded as it scrolls (shows about 6-8 at a time), so it scrolls, scrapes, appends to a dataframe, then repeats. It works great for the first few hundred rows, then it starts to get duplicate rows. Any idea what I'm doing wrong?
def scrapenotis():
driver.get("NOTIFICATIONS");
WebDriverWait(driver, 120).until(EC.presence_of_element_located((By.CSS_SELECTOR, "div.vue-recycle-scroller__item-view")));
tbltitles = ["Datetime","Username","User Link","Description","Category","Desc Link"];
tablelist = [];
starttime = datetime.now()
while driver.title == "WEBSITE TITLE":
try:
# gets list of all entries
entries = driver.find_elements(By.CSS_SELECTOR, "div.vue-recycle-scroller__item-view");
if len(entries) == 0:
break;
# iterates through entries
for x in entries:
# checking for those elements that persist for some ungodly reason
if x.get_attribute("style") == "transform: translateY(-9999px);":
continue;
#each entry is 83 pixels long with about 6 on screen at a time
driver.execute_script("window.scrollBy(0, 300);");
# entries need to load after scroll, they load twice within a second(?)
time.sleep(1.5);
# datedesc = driver.find_element(By.XPATH, "//*[@id='content']/div[1]/div[1]/div/div[3]/div/div[1]//div/div/div[5]/span/span").get_attribute("title");
datedesc = driver.find_element(By.CSS_SELECTOR, "span.b-notifications__list__item__actions__item.g-date span").get_attribute("title");
username = driver.find_element(By.CSS_SELECTOR, "div.b-username-wrapper div.g-user-name").text;
userlink = driver.find_element(By.CSS_SELECTOR, "div.b-username-wrapper a").get_attribute("href");
description = driver.find_element(By.CSS_SELECTOR, "div.b-notifications__list__item__text div.g-truncated-text").text;
#sorting them out for categories
if "ubscribed" in description:
cat = "New Sub";
desclink = "N/A";
elif "iked your" in description:
cat = "Like";
desclink = driver.find_element(By.CSS_SELECTOR, "div.b-notifications__list__item__text div.g-truncated-text a").get_attribute("href");
elif "restarted their monthly subscription" in description:
cat = "Sub Renewal";
desclink = "N/A";
elif "purchased your" in description:
cat = "Purchase";
desclink = driver.find_element(By.CSS_SELECTOR, "div.b-notifications__list__item__text div.g-truncated-text a").get_attribute("href");
elif any(x in description for x in ["eplied","esponded"]):
cat = "Comment";
desclink = driver.find_element(By.CSS_SELECTOR, "div.b-notifications__list__item__text div.g-truncated-text a").get_attribute("href");
elif "tip" in description:
cat = "Tip";
desclink = "N/A";
dict1 = [datedesc,username,userlink,description,cat,desclink];
tablelist.append(dict1);
#specify stop time in seconds
if (datetime.now()-starttime).seconds >= 14400: #3600(1 hour) * 4 = 14400
break;
except:
break
#convert list to df
msgbox(tablelist);
df = pd.DataFrame(tablelist,columns=tbltitles);
df.drop_duplicates(subset=tbltitles, inplace=True, keep='first');
#save to csv
path = filesavebox("Save your updated data file","","",["*.txt","*.csv"]);
if path == None:
return;
df.to_csv(path + ".csv");