how to save the result of webs craping in the out put file text with selenium in python

user_16832397 0
Viewed 43

the code below retrieve all the data from the url below. But I want to save the results in the output file text. like (info.text), Would you mind help me.

url: https://www150.statcan.gc.ca/n1/pub/71-607-x/2021004/exp-eng.htm?r1=(1)&r2=0&r3=0&r4=12&r5=0&r7=0&r8=2022-02-01&r9=2022-02-01

from selenium import webdriver
import time

url = "https://www150.statcan.gc.ca/n1/pub/71-607-x/2021004/exp-eng.htm?r1=(1)&r2=0&r3=0&r4=12&r5=0&r7=0&r8=2022-02-01&r9=2022-02-01"

driver = webdriver.Chrome("C:\Program Files\Python310\chromedriver.exe")
driver.get(url)
table = driver.find_element_by_id('report_table')
body = table.find_element_by_tag_name('tbody')
cells = body.find_elements_by_tag_name('td')

for cell in cells:
    print(cell.text)

# run the loop 26 times
for i in range(26):
    
    # your code
    table = driver.find_element_by_id('report_table')
    body = table.find_element_by_tag_name('tbody')
    cells = body.find_elements_by_tag_name('td')
    for cell in cells:
        print(cell.text)
        
    # click on the Next button    
    driver.find_element_by_xpath('//*[@id="report_results_next"]').click()
selenium python
2 Answers

If you want to write the results from a variable (seems like cell.text in your example), we just need to open a file and write that data to the file.

with open("info.txt", “w”, encoding = 'utf-8') as f:
   f.write(cell.txt)

This is making an assumption that cell.text is a string, by the way

user_81055140

This may help, you can read further about file handling here

from bs4 import BeautifulSoup
from selenium import webdriver
import pandas as pd
import time
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
from selenium import webdriver
import time


options = Options()
options.add_experimental_option("w3c", True)
options = webdriver.ChromeOptions()

f = open("datafile.txt", "a")

url = "https://www150.statcan.gc.ca/n1/pub/71-607-x/2021004/exp-eng.htm?r1=(1)&r2=0&r3=0&r4=12&r5=0&r7=0&r8=2022-02-01&r9=2022-02-01"

driver = webdriver.Chrome(ChromeDriverManager().install(),options=options)

driver.get(url)
table = driver.find_element_by_id('report_table')
body = table.find_element_by_tag_name('tbody')
cells = body.find_elements_by_tag_name('td')

for cell in cells:
    #print(cell.text)
    f.write(cell.text)

# run the loop 26 times
for i in range(26):
    
    # your code
    table = driver.find_element_by_id('report_table')
    body = table.find_element_by_tag_name('tbody')
    cells = body.find_elements_by_tag_name('td')
    for cell in cells:
        #print(cell.text)
        f.write(cell.text)
        
    # click on the Next button    
    driver.find_element_by_xpath('//*[@id="report_results_next"]').click()
f.close()
user_175777220
Related