How can I make my natural logarithm calculation from dataframe columns automatic?

user_15322377 0
Viewed 60

I have to create columns that are the calculation of the natural log of other columns in the data set. There are too many columns (features) and I would like to make it automatic, but the for loop I have tried did not work. Here is the list of columns that I have called 'features':

features=['price_seat',
              'days_length_of_stay',
              'days_to_departure',
              'distance',
              'unit_cost_brute',
              'unit_cost_clip',
              'unit_cost_mean',
              'unit_cost',
              'org_country_gdp_per_capita',
              'dst_country_gdp_per_capita',
              'competing_airline',
              #'yield',
              'price_seat_cluster',
              'yield_cluster',
              'low_cost',
              #'PAX',
              #'REVENUE',
              'LOCAL_PAX',
              'BEHIND_PAX',
              'BEYOND_PAX',
              'BRIDGE_PAX',
              'LOCAL_REVENUE',
              'BEHIND_REVENUE',
              'BEYOND_REVENUE',
              'BRIDGE_REVENUE',
              'REVENUE_WITH_TAXES',
              'LOCAL_REVENUE_WITH_TAXES',
              'BRIDGE_REVENUE_WITH_TAXES',
              'BEHIND_REVENUE_WITH_TAXES',
              'BEYOND_REVENUE_WITH_TAXES',
              'PERIOD',
              'n_flights_month',
              'avg_flights_month',
              'flights_month',
              #'pax_flight',
              'revenue_flight',
              #'revenue_pax',
              'WTI',
              'Brent',
              'Jet_fuel',
              'OilPrice_USD_bbl',
              'FuelPrice_USD_USgal',
              'Density',
              'Cf_USD_kg',
              'd_fr24',
              'distance_fr']

And here is the code I have used and it works:

 df=df9.withColumn('ln_price_seat', F.log('price_seat'))\
    .withColumn('ln_days_length_of_stay',F.log('days_length_of_stay'))\
    .withColumn('ln_days_to_departure',F.log('days_to_departure'))\
    .withColumn('ln_distance',F.log('distance'))\
    .withColumn('ln_unit_cost_brute',F.log('unit_cost_brute'))\
    .withColumn('ln_unit_cost_clip',F.log('unit_cost_clip'))\
    .withColumn('ln_unit_cost_mean',F.log('unit_cost_mean'))

But this is too 'manual' for so many features and I might change those features in the future, so I need something that can handle that. On top of that my dataframe is quite large, in the order of 50M or more. I was able to do this process before doing this:

def get_log_features(self,df):

    
    features=['price_seat',
              'days_length_of_stay',
              'days_to_departure',
              'distance',
              'unit_cost_brute',
              'unit_cost_clip',
              'unit_cost_mean',
              'unit_cost',
              'org_country_gdp_per_capita',
              'dst_country_gdp_per_capita',
              'competing_airline',
              'price_seat_cluster',
              'yield_cluster',
              'low_cost',
              'LOCAL_PAX',
              'BEHIND_PAX',
              'BEYOND_PAX',
              'BRIDGE_PAX',
              'LOCAL_REVENUE',
              'BEHIND_REVENUE',
              'BEYOND_REVENUE',
              'BRIDGE_REVENUE',
              'REVENUE_WITH_TAXES',
              'LOCAL_REVENUE_WITH_TAXES',
              'BRIDGE_REVENUE_WITH_TAXES',
              'BEHIND_REVENUE_WITH_TAXES',
              'BEYOND_REVENUE_WITH_TAXES',
              'PERIOD',
              'n_flights_month',
              'avg_flights_month',
              'flights_month',
              'revenue_flight',
              'WTI',
              'Brent',
              'Jet_fuel',
              'OilPrice_USD_bbl',
              'FuelPrice_USD_USgal',
              'Density',
              'Cf_USD_kg',
              'd_fr24',
              'distance_fr']
    


    features_for_log=features
    df_log= (df.select(*features_for_log,'org_airport','dst_airport','d_year','d_month'))
    
    for new_col in features_for_log:
        df_log = df_log.withColumn('ln_'+ new_col, F.log(F.col(new_col)))
        
    df_log= (df_log.drop(*features_for_log))
    
    
    df=(df.join(df_log,['org_airport','dst_airport','d_year','d_month'],how='outer'))

But when I call this function it takes hours, its too computational expensive, thats why I thought on 'appending' the original dataframe with the natural log of the columns defined by the feature list, and that could be less expensive.

Do you have any recommendations?

bigdata pyspark python
2 Answers

The easiest and fastest way would be what you have already described: add the log columns to the dataframe:

cols = [F.col(col) for col in df.columns]
ln_cols = [F.log(col).alias(f"ln_{col}") for col in features_for_log]
df = df.select(cols + ln_cols)
user_21298010

I did it like this and it worked:

from pyspark.sql.types import *

from pyspark.sql import functions as F

class GetLogFeatures:

def __init__(self,spark):
    self._spark_=spark
    
def get_log_features(self,df):

    
    features=['price_seat',
              'days_length_of_stay',
              'days_to_departure',
              'distance',
              'unit_cost_brute',
              'unit_cost_clip',
              'unit_cost_mean',
              'unit_cost',
              'org_country_gdp_per_capita',
              'dst_country_gdp_per_capita',
              'competing_airline',
              'price_seat_cluster',
              'yield_cluster',
              'low_cost',
              'LOCAL_PAX',
              'BEHIND_PAX',
              'BEYOND_PAX',
              'BRIDGE_PAX',
              'LOCAL_REVENUE',
              'BEHIND_REVENUE',
              'BEYOND_REVENUE',
              'BRIDGE_REVENUE',
              'REVENUE_WITH_TAXES',
              'LOCAL_REVENUE_WITH_TAXES',
              'BRIDGE_REVENUE_WITH_TAXES',
              'BEHIND_REVENUE_WITH_TAXES',
              'BEYOND_REVENUE_WITH_TAXES',
              'PERIOD',
              'n_flights_month',
              'avg_flights_month',
              'flights_month',
              'revenue_flight',
              'WTI',
              'Brent',
              'Jet_fuel',
              'OilPrice_USD_bbl',
              'FuelPrice_USD_USgal',
              'Density',
              'Cf_USD_kg',
              'd_fr24',
              'distance_fr']
    
    for new_col in features:
        df = df.withColumn('ln_'+ new_col, F.log(F.col(new_col)))
    
    return df
user_153223770
Related