Handling Firebase 'Max retries exceeded with url' error in Python

user_14426785 0
Viewed 325

I've been using the python-firebase library to upload some data to my realtime database with the POST method from the package. After a few 'products' where uploaded the following error shows up, i've already tried to use wait a few seconds between every POST.

> Traceback (most recent call last):
  File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/requests/packages/urllib3/connection.py", line 141, in _new_conn
    conn = connection.create_connection(
  File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/requests/packages/urllib3/util/connection.py", line 98, in create_connection
    raise err
  File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/requests/packages/urllib3/util/connection.py", line 88, in create_connection
    sock.connect(sa)
ConnectionRefusedError: [Errno 61] Connection refused

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/requests/packages/urllib3/connectionpool.py", line 592, in urlopen
    httplib_response = self._make_request(conn, method, url,
  File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/requests/packages/urllib3/connectionpool.py", line 352, in _make_request
    self._validate_conn(conn)
  File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/requests/packages/urllib3/connectionpool.py", line 831, in _validate_conn
    conn.connect()
  File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/requests/packages/urllib3/connection.py", line 254, in connect
    conn = self._new_conn()
  File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/requests/packages/urllib3/connection.py", line 150, in _new_conn
    raise NewConnectionError(
requests.packages.urllib3.exceptions.NewConnectionError: <requests.packages.urllib3.connection.VerifiedHTTPSConnection object at 0x7fe562582220>: Failed to establish a new connection: [Errno 61] Connection refused

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/requests/adapters.py", line 413, in send
    resp = conn.urlopen(
  File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/requests/packages/urllib3/connectionpool.py", line 639, in urlopen
    retries = retries.increment(method, url, error=e, _pool=self,
  File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/requests/packages/urllib3/util/retry.py", line 287, in increment
    raise MaxRetryError(_pool, url, error or ResponseError(cause))
requests.packages.urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='listy-itba-app.firebaseio.com', port=443): Max retries exceeded with url: /products/.json (Caused by NewConnectionError('<requests.packages.urllib3.connection.VerifiedHTTPSConnection object at 0x7fe562582220>: Failed to establish a new connection: [Errno 61] Connection refused'))

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/Users/roberto-j-catalan/CotoScraper/MerginJson.py", line 29, in <module>
    firebase.post('/products', product)
  File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/firebase/decorators.py", line 19, in wrapped
    return f(*args, **kwargs)
  File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/firebase/firebase.py", line 328, in post
    return make_post_request(endpoint, data, params, headers,
  File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/firebase/decorators.py", line 19, in wrapped
    return f(*args, **kwargs)
  File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/firebase/firebase.py", line 96, in make_post_request
    response = connection.post(url, data=data, params=params, headers=headers,
  File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/requests/sessions.py", line 522, in post
    return self.request('POST', url, data=data, json=json, **kwargs)
  File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/requests/sessions.py", line 475, in request
    resp = self.send(prep, **send_kwargs)
  File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/requests/sessions.py", line 596, in send
    r = adapter.send(request, **kwargs)
  File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/requests/adapters.py", line 487, in send
    raise ConnectionError(e, request=request)
requests.exceptions.ConnectionError: HTTPSConnectionPool(host='listy-itba-app.firebaseio.com', port=443): Max retries exceeded with url: /products/.json (Caused by NewConnectionError('<requests.packages.urllib3.connection.VerifiedHTTPSConnection object at 0x7fe562582220>: Failed to establish a new connection: [Errno 61] Connection refused'))

And this is my code so far:

import json
from firebase import firebase
import re
from time import sleep
firebase = firebase.FirebaseApplication('website_url')
if __name__ == '__main__':

    with open('./coto_scraper/data.json') as file:
        data = json.load(file)
        products = []
    for page in data:

        names = page["name"]
        prices = page["price"]
        cant_names = len(names)
        cant_prices = len(prices)
        print('cant name: ' + str(cant_names))
        print('cant price: ' + str(cant_prices))
        cant_products = cant_names
        make_change = False
        if cant_prices < cant_names:
            cant_products = cant_prices
            make_change = True
        for i in range(cant_products):
            true_name = re.sub(' +', ' ', names[i].replace('\n','').replace('\r','').replace('\t', '').rstrip())
            true_name = re.sub('( \.)+', '', true_name)
            product = {'name': true_name, 'price': prices[i].replace('\n', '').replace('\r', '').replace('\t', '')}
            products.append(product)
            firebase.post('/products', product)
            print(product)
        if make_change:
            product = {'name': names[cant_names - 1], 'price': -1}
    with open('products.json', 'w', encoding='utf-8') as json_file:
        json.dump(products, json_file, ensure_ascii=False)
firebase-realtime-database firebase python
1 Answers

Use Firebase-Admin for Python

  • This is usually faster than REST API
  • Can run multiple threads and update the database simultaneously
pip install firebase-admin

Download the Admin SDK as json

  • To generate a private key file for your service account:

    • In the Firebase console, open Settings > Service Accounts.
    • Click Generate New Private Key, then confirm by clicking Generate Key.
    • Securely store the JSON file containing the key.
    • For more details checkout Firebase admin sdk
    • Save the file as firebase-admin-key.json

Sample Code


#Firebase Imports
import firebase_admin
from firebase_admin import credentials
from firebase_admin import db

cred = credentials.Certificate("/firebase-admin-key.json")


print("initilising firebase app")

#Initilising Database
firebase_admin.initialize_app(cred, {
    'databaseURL' : 'https://<project-id>.firebaseio.com/'
})

# Initilising Database 2
app2 = firebase_admin.initialize_app(cred, {
    'databaseURL': 'https://<project-id>.firebaseio.com/'
}, name='app2')


# Update Data from firebase
def update_data_firebase(path,data):
    ref=db.reference('items',app2).child(path)
    ref.set(data)
    print("Succesfully updated the data in firebase")

# Main Function
def main():
    data_to_update={"my-key": "value"}
    update_data_firebase("Updated",data_to_update)

#Main program
if __name__ == '__main__':
    main()

Similar Question Python Requests Max retries exceeded with URL for Firebase Database REST API

user_123133210
Related