I am using a combination of Flask and Pandas to return a created Excel file. The data is read in as chunks using the Pandas read_sql method like so:
chunks = pd.read_sql(query_string, conn_sql_server, chunksize=1000)
As the returning results from the query can be very large (around 800,000 rows) I make use of Flask's Streaming with Context
return Response(stream_with_context(generate_function(data_chunk, conn_sql_server)), headers=headers)
Where my generate_function is as follows:
def generate_function(self, data_chunk, conn_sql_server):
logger.info('Reading all SQL Server Table')
try:
is_first_iteration = True
response = ''
is_complete = False
total_rows = 0
output = BytesIO()
writer = pd.ExcelWriter(output, engine='xlsxwriter')
for chunk in data_chunk:
if total_rows == self.top_data or total_rows >= self.limit_rows_xlsx or total_rows == self.num_total_count:
writer.save()
output.seek(0)
response = output.getvalue()
yield response
chunk.to_excel(writer, index=False,
sheet_name='Sheet1', encoding='utf-8')
total_rows = total_rows + len(chunk)
yield chunk
except Exception as e:
logger.error(f'Error formatting dataframe to xlsx: {str(e)}')
raise FormatDataframeToXlsx('Error formatting data to xlsx')
finally:
if conn_sql_server is not None:
conn_sql_server.close()
However this returns me nothing. The only other option I thought was to loop the chunks and create a dataframe and then yield the result however this occurs an Nginx timeout on the server:
is_first_iteration = True
response = ''
is_complete = False
total_rows = 0
for chunk in data_chunk:
if is_complete:
break
if is_first_iteration:
data = chunk
is_first_iteration = False
else:
data = data.append(chunk)
total_rows = total_rows + len(chunk)
if total_rows == self.top_data or total_rows >= self.limit_rows_xlsx or total_rows == self.num_total_count:
if total_rows >= self.limit_rows_xlsx:
is_complete = True
data = data[:self.limit_rows_xlsx]
output = BytesIO()
writer = pd.ExcelWriter(output, engine='xlsxwriter')
data.to_excel(writer, index=False,
sheet_name='Sheet1', encoding='utf-8')
writer.save()
output.seek(0)
response = output.getvalue()
yield response
Is there a more suited method to handling large amounts of data to be written to excel like this?