I´m using Google Cloud Funtion to create a table, for now everything works the way it is supposed to.
But I would like to add a new field in the table, one that could show the time of its creation.
This is an example of the code that I´m using at the moment.
Don´t know why Is not working but my main goal is to actually be able to do it with one table and the replicate the process in codes there I handle two or more tables.
Example:
structure of the data in the bucket:
Function
Code
main
from google.cloud
import bigquery
import pandas as pd
from previsional_tables
import table_TEST1
creation_date = pd.Timestamp.now()# Here is where I´ m supposed to get the date.
def main_function(event, context):
dataset = 'bd_clients'
file = event
input_bucket_name = file['bucket']
path_file = file['name']
uri = 'gs://{}/{}'.format(input_bucket_name, path_file)
path_file_list = path_file.split("/")
file_name_ext = path_file_list[len(path_file_list) - 1]
file_name_ext_list = file_name_ext.split(".")
name_file = file_name_ext_list[0]
print('nombre archivo ==> ' + name_file.upper())
print('Getting the data from bucket "{}"'.format(uri))
path_file_name = str(uri)
print("ruta: ", path_file_name)
if ("gs://bucket_test" in path_file_name):
client = bigquery.Client()
job_config = bigquery.LoadJobConfig()
table_test1(dataset, client, uri, job_config, bigquery)
tables
def table_test1(dataset, client, uri, job_config, bigquery):
table = "test1"
dataset_ref = client.dataset(dataset)
job_config.autodetect = True
job_config.max_bad_records = 1000
job_config.schema = [
bigquery.SchemaField("NAME", "STRING"),
bigquery.SchemaField("LAST_NAME", "STRING"),
bigquery.SchemaField("ADDRESS", "STRING"),
bigquery.SchemaField("DATE", bigquery.enums.SqlTypeNames.DATE)# create each column in Big Query along with types
]
job_config.source_format = bigquery.SourceFormat.CSV
job_config.field_delimiter = ';'
job_config.write_disposition = bigquery.WriteDisposition.WRITE_APPEND
load_job = client.load_table_from_uri(uri, dataset_ref.table(table), job_config = job_config)
requirements
# Function dependencies, for example:
# package>=version
google-cloud-bigquery==2.25.1
pysftp==0.2.9
pandas==1.4.2
fsspec==2022.5.0
gcsfs==2022.5.0
Output structure in Database:


