I have the following Python (2.7) function using Pandas which I need to run on 400 GB. A test run on 150 GB took 4 hours to complete successfully (memory on machine is 128 GB and 16 cores, 4TB disk). Is there a smart way to utilize all the cores on my CPU for this and doing this action in parallel to decrease the process time:
def create_data(headers, filters, filesFolder, remove_chars, outputFolder):
for ds in headers:
for data_file in glob.glob(filesFolder + '*' + ds + '*.csv'):
x=0
for data in pd.read_csv(data_file, sep = '€', names = headers[ds], engine = 'python', chunksize = 10000000):
logger.info('Pandas Read %s' %(str(x) + '_' + os.path.basename(data_file)) )
for fil in filters[ds]:
try:
data[fil] = data[fil].astype('O')
data = data[data[fil] == filters[ds][fil]]
logger.info('Filtered data for %s succesfully. Filters are %s' %(str(x) + '_' + os.path.basename(data_file) , filters[ds]))
except:
logger.info('Could not apply filter %s for %s' %(filters[ds][fil], str(x) + '_' + os.path.basename(data_file) ))
pass
for char in remove_chars:
try:
data = data.replace({char: ''}, regex=True)
logger.info('Removed char %s from %s succesfully' %(char, str(x) + '_' + os.path.basename(data_file) ))
except:
logger.info('Could not remove char %s from %s' %(char, str(x) + '_' + os.path.basename(data_file) ))
pass
try:
data.to_csv(outputFolder + ds + '/' + str(x) + '_' + os.path.basename(data_file).split('.')[0].strip() + '.csv', enconding = 'utf-8')
logger.info('Saved csv %s succesfully' %(str(x) + '_' + os.path.basename(data_file)))
x+=1
except:
logger.info('Failed to Save %s' %(str(x) + '_' + os.path.basename(data_file)))