Asynchronous data recording with SQLAlchemy by iterating through pandas Dataframe

Viewed 14

I am working on a function which accepts 2 parameters: data (pandas Dataframe) and source of information (dict). This function records information asynchronously to the database using SQLAlchmey iterating through the data frame, one by one row.

I fairly new to asynchronous coding and I feel that my function has problem with the loop. Although code inside the loop seems to be asynchronous the loop itself probably blocks the process. Also I see the potential problem with Source object. We shall make sure that it is created in advance, before we go to loop, because we need to use this object for model relations inside the loop.

Given the above I need advice/guidance:

  1. how to make the loop asynchronous
  2. how to make sure that in asynchronous approach Source object will be created before we move to the loop (probably need to make code synchronous for this task)
  3. ideas on how to make the process of recoding data frame into the database more efficient than iterating through dataframe rows

Code

models.py

from sqlalchemy import Column, String, Date, ForeignKey, Integer, Boolean
from sqlalchemy.orm import relationship
from datetime import datetime

from data_parser.core.base import Base


class Dosar(Base):
    dosar_number = Column(String(252))
    application_date = Column(Date)
    termens = relationship(
        'Termen',
        back_populates='dosar',
        cascade='all, delete-orphan',
        passive_deletes=True
    )
    solutions = relationship(
        'Solution',
        back_populates='dosar',
        cascade='all, delete-orphan',
        passive_deletes=True
    )

class Source(Base):
    file_name = Column(String(252))
    last_modified_date = Column(Date)
    version = Column(String(10))
    file_size = Column(Integer)
    parsing_date = Column(Date, default=datetime.now)
    termens = relationship(
        'Termen',
        back_populates='source',
        cascade='all, delete-orphan',
        passive_deletes=True
    )
    solutions = relationship(
        'Solution',
        back_populates='source',
        cascade='all, delete-orphan',
        passive_deletes=True
    )

class Solution(Base):
    solution_act = Column(String(252))
    solution_date = Column(Date)
    solution_true_date = Column(Boolean, default=True)
    dosar_id = Column(
        Integer,
        ForeignKey('dosar.id', ondelete='CASCADE'))
    source_id = Column(
        Integer,
        ForeignKey('source.id', ondelete='CASCADE'))
    dosar = relationship('Dosar', back_populates='solutions')
    source = relationship('Source', back_populates='solutions')

class Termen(Base):
    termen_date = Column(Date)
    dosar_id = Column(
        Integer,
        ForeignKey('dosar.id', ondelete='CASCADE'))
    source_id = Column(
        Integer,
        ForeignKey('source.id', ondelete='CASCADE'))
    dosar = relationship('Dosar', back_populates='termens')
    source = relationship('Source', back_populates='termens')

function.py

import pandas as pd
import asyncio

from data_parser.crud.dosar import dosar_crud
from data_parser.crud.source import source_crud
from data_parser.crud.termen import termen_crud
from data_parser.crud.solution import solution_crud
from data_parser.schemas.dosar import DosarData #pydantic schema
from data_parser.schemas.source import SourceData #pydantic schema
from data_parser.schemas.termen import TermenData #pydantic schema
from data_parser.schemas.soultion import SolutionData #pydantic schema
from data_parser.core.db import AsyncSessionLocal, engine
from data_parser.core.db import Base


async def data_recording(dataframe: pd.DataFrame,
                         source: dict
                         ) -> None:
    async with engine.begin() as conn:
        await conn.run_sync(Base.metadata.create_all)

    async with AsyncSessionLocal() as session:
        source_data = SourceData(**source)
        source = await source_crud.get_source_by_attrs_or_create(session,
                                                                 source_data)
    for row in dataframe.itertuples():
        async with AsyncSessionLocal() as session:
            dosar_data = DosarData(**row._asdict())  # noqa
            dosar = await dosar_crud.get_dosar_by_name_or_create(session,
                                                                 dosar_data)
            if not pd.isnull(row.termen_date):
                termen_data = TermenData(**row._asdict(),  # noqa
                                         dosar_id=dosar.id,
                                         source_id=source.id)
                await termen_crud.create(termen_data, session)

            if not pd.isnull(row.solution_act):
                solution_data = SolutionData(**row._asdict(),
                                             dosar_id=dosar.id,
                                             source_id=source.id)
                await solution_crud.create(solution_data, session)
0 Answers
Related