Can't find any idea to print data from mysql using function

Viewed 275

I've created a script in python capable of collecting data from a webpage and storing the same into mysql. When the data are properly inserted into mysql, my script can, however, print them in the console.

My question is: how can I wrap the following three lines within a seperate function and print the data from the storage?

mycursor.execute("SELECT * FROM webdata")
for item in mycursor.fetchall():
    print(item)

My full script:

import mysql.connector
from bs4 import BeautifulSoup
import requests

URL = "https://www.tripadvisor.com.au/Restaurants-g255068-c8-Brisbane_Brisbane_Region_Queensland.html"

def get_info(link):
    mydb = mysql.connector.connect(
      host="localhost",
      user="root",
      passwd = "123",
      database="mydatabase"
    )
    mycursor = mydb.cursor()
    mycursor.execute("DROP TABLE if exists webdata")
    mycursor.execute("CREATE TABLE if not exists webdata (name VARCHAR(255), bubble VARCHAR(255), review VARCHAR(255))")

    response = requests.get(link)
    soup = BeautifulSoup(response.text,"lxml")
    for items in soup.find_all(class_="shortSellDetails"):
        name = items.find(class_="property_title").get_text(strip=True)
        bubble = items.find(class_="ui_bubble_rating").get("alt")
        review = items.find(class_="reviewCount").get_text(strip=True)

        mycursor.execute("INSERT INTO webdata (name,bubble,review) VALUES (%s,%s,%s)",(name,bubble,review))
        mydb.commit()

    #I wish to use the follwing three lines within another function to do the same

    mycursor.execute("SELECT * FROM webdata")
    for item in mycursor.fetchall():
        print(item)

if __name__ == '__main__':
    get_info(URL)
3 Answers

My comment is meant to describe this changed code:

import mysql.connector
from bs4 import BeautifulSoup
import requests

# This is a new function
def do_fetch (cursor):
    cursor.execute("SELECT * FROM webdata")
    for item in cursor.fetchall():
        print(item)

URL = "https://www.tripadvisor.com.au/Restaurants-g255068-c8-Brisbane_Brisbane_Region_Queensland.html"

def get_info(link):
    mydb = mysql.connector.connect(
      host="localhost",
      user="root",
      passwd = "123",
      database="mydatabase"
    )
    mycursor = mydb.cursor()
    mycursor.execute("DROP TABLE if exists webdata")
    mycursor.execute("CREATE TABLE if not exists webdata (name VARCHAR(255), bubble VARCHAR(255), review VARCHAR(255))")

    response = requests.get(link)
    soup = BeautifulSoup(response.text,"lxml")
    for items in soup.find_all(class_="shortSellDetails"):
        name = items.find(class_="property_title").get_text(strip=True)
        bubble = items.find(class_="ui_bubble_rating").get("alt")
        review = items.find(class_="reviewCount").get_text(strip=True)

        mycursor.execute("INSERT INTO webdata (name,bubble,review) VALUES (%s,%s,%s)",(name,bubble,review))
        mydb.commit()

    # This part is changed (moved to new function):
    do_fetch (mycursor)

if __name__ == '__main__':
    get_info(URL)

I think the following is how you can seperate things into different functions and print the data as required. Make sure to commit outside of the for loop like how it is suggested here.

import mysql.connector
from bs4 import BeautifulSoup
import requests

url = "https://www.tripadvisor.com.au/Restaurants-g255068-c8-Brisbane_Brisbane_Region_Queensland.html"

def connect():
    mydb = mysql.connector.connect(
      host="localhost",
      user="root",
      passwd = "123",
      database="mydatabase"
    )
    return mydb

def create_table(link):
    conn = connect()
    mycursor = conn.cursor()
    mycursor.execute("DROP TABLE if exists webdata")
    mycursor.execute("CREATE TABLE if not exists webdata (name VARCHAR(255), bubble VARCHAR(255), review VARCHAR(255))")

    response = requests.get(link)
    soup = BeautifulSoup(response.text,"lxml")
    for items in soup.find_all(class_="shortSellDetails"):
        name = items.find(class_="property_title").get_text(strip=True)
        bubble = items.find(class_="ui_bubble_rating").get("alt")
        review = items.find(class_="reviewCount").get_text(strip=True)
        mycursor.execute("INSERT INTO webdata (name,bubble,review) VALUES (%s,%s,%s)",(name,bubble,review))
    conn.commit() #this is where you should commit

def fetch_data():
    conn = connect()
    mycursor = conn.cursor()
    mycursor.execute("SELECT * FROM webdata")
    for item in mycursor.fetchall():
        print(item)

if __name__ == '__main__':
    create_table(url)
    fetch_data()

Have you considered using the Django model framework it allows you interact with any table as a python object.

in an existing db you can convert it to a model as described here, this would simply be a matter of setting up your db in django settings file

DATABASES = {
'default': {
    'ENGINE': 'django.db.backends.mysql', 
    'NAME': 'DB_NAME',
    'USER': 'DB_USER',
    'PASSWORD': 'DB_PASSWORD',
    'HOST': 'localhost',   # Or an IP Address that your DB is hosted on
    'PORT': '3306',
    }
}

you could then use the django ops to create a models from the tables

$ python manage.py inspectdb > models.py

you'll then end up with some object definition in your models.py file which you can then access.

You'll end up with some object (without knowing too much about your db) like this, to which you just need to add a __str__() handle to, so that when you print the object it's formatted correctly:

from django.db import models

class ShortSellDetails(models.Model):
    id = models.UUIDField(default=uuid.uuid4, editable=False, unique=True)
    name = models.CharField()
    bubble = models.CharField()
    review = models.TextField()

    def __str__(self):
        return "VALUES ({0},{1}, {2})".format(
             self.name,self.bubble,self.review)

you can then interact with this object like all models (make sure you add your model to your INSTALLED APPS e.g. shortSell and perform ./manage.py makemigrations & ./manage.py migrate) eg:

from shortSell import ShortSellDetails

details = ShortSellDetails.object.all()
for detail in details:
    print(detail)

Or you could also just create the objects directly using the model interface

from bs4 import BeautifulSoup
import requests
from shortSell import ShortSellDetails

URL = "https://www.tripadvisor.com.au/Restaurants-g255068-c8-Brisbane_Brisbane_Region_Queensland.html"
response = requests.get(URL)
soup = BeautifulSoup(response.text,"lxml")

for items in soup.find_all(class_="shortSellDetails"):
        name = items.find(class_="property_title").get_text(strip=True)
        bubble = items.find(class_="ui_bubble_rating").get("alt")
        review = items.find(class_="reviewCount").get_text(strip=True)


        ShortSellDetails.objects.create(
            name=name, 
            bubble=bubble, 
            review=review)

details = ShortSellDetails.object.all()
for detail in details:
    print(detail)
Related