Subtract a dataframe with some matching and non matching columns and indexes

Viewed 522

How can I subtract two dataframes that have some matching and some non-matching columns and indexes?

df_diff = df_add - df_subtract
df_diff = df_add.subtract(df_subtract)

where:

df_add:

    1   2   3   4
A   1.1 1.2 1.3 1.4
B   2.1 2.2 2.3 2.4
D   3.1 3.2 3.3 3.4
E   4.1 4.2 4.3 4.4

df_subtract:

    2   4
B   5   8
C   6   9
D   7   10

trying to get: df_diff:

    1     2      3       4
A   1.1   1.2    1.3     1.4
B   2.1   -2.8   2.3    -5.6
C   0     -6     0       -9
D   3.1   -3.8   3.3    -6.6
E   4.1   4.2    4.3    4.4
3 Answers

Use pd.DataFrame.sub with fill_value, then fillna for missing values in df_add dataframe:

df_add.sub(df_sub, fill_value=0).fillna(0)

Output:

     1    2    3    4
A  1.1  1.2  1.3  1.4
B  2.1 -2.8  2.3 -5.6
C  0.0 -6.0  0.0 -9.0
D  3.1 -3.8  3.3 -6.6
E  4.1  4.2  4.3  4.4

Try reindex to uniform the shapes of both DataFrames then do normal subtraction:

# Get All Index
new_idx = df_add.index.union(df_subtract.index)
# Get All Columns
new_cols = df_add.columns.union(df_subtract.columns)
df_diff = (
        df_add.reindex(index=new_idx, columns=new_cols, fill_value=0)
        -
        df_subtract.reindex(index=new_idx, columns=new_cols, fill_value=0)
)

Reshaped df_add:

     1    2    3    4
A  1.1  1.2  1.3  1.4
B  2.1  2.2  2.3  2.4
C  0.0  0.0  0.0  0.0
D  3.1  3.2  3.3  3.4
E  4.1  4.2  4.3  4.4

Reshaped df_subtract:

   1  2  3   4
A  0  0  0   0
B  0  5  0   8
C  0  6  0   9
D  0  7  0  10
E  0  0  0   0

df_diff:

     1    2    3    4
A  1.1  1.2  1.3  1.4
B  2.1 -2.8  2.3 -5.6
C  0.0 -6.0  0.0 -9.0
D  3.1 -3.8  3.3 -6.6
E  4.1  4.2  4.3  4.4

Timing Information Via Perfplot:

benchmarking perfplot

import numpy as np
import pandas as pd
import perfplot

np.random.seed(5)


def gen_data(n):
    df_add = pd.DataFrame(np.random.random(size=(n, n)))
    df_subtract = pd.DataFrame(np.random.random(size=(n, n))) \
        .sample(frac=.5).sample(frac=.5, axis=1) \
        .sort_index().sort_index(axis=1)
    if df_subtract.empty:
        return df_add, df_subtract
    return (
        df_add.drop(np.random.choice(df_subtract.index,
                                     max(1, int(df_subtract.shape[0] * .2)))),
        df_subtract
    )


def reindex(dfs):
    df_add, df_subtract = dfs
    new_idx = df_add.index.union(df_subtract.index)
    new_cols = df_add.columns.union(df_subtract.columns)
    return (
            df_add.reindex(index=new_idx, columns=new_cols, fill_value=0)
            -
            df_subtract.reindex(index=new_idx, columns=new_cols, fill_value=0)
    )


def sub(dfs):
    df_add, df_subtract = dfs
    return df_add.sub(df_subtract, fill_value=0).fillna(0)


def combine_first(dfs):
    df_add, df_subtract = dfs
    return (df_add - df_subtract) \
        .combine_first(df_add) \
        .combine_first(df_subtract) \
        .fillna(0)


if __name__ == '__main__':
    out = perfplot.bench(
        setup=gen_data,
        kernels=[
            sub,
            reindex,
            combine_first
        ],
        labels=[
            'sub @ScottBoston',
            'reindex @HenryEcker',
            'combine_first @DYZ'
        ],
        n_range=[2 ** k for k in range(15)],
        equality_check=None
    )
    out.save('perfplot_results.png', transparent=False)

First, take the difference, where possible. Then patch the missing values using the data from the original DataFrames. Finally, fill the remaining missing values with 0s.

(df_add - df_subtract)\
    .combine_first(df_add)\
    .combine_first(df_subtract)\
    .fillna(0)
#     1    2    3    4
#A  1.1  1.2  1.3  1.4
#B  2.1 -2.8  2.3 -5.6
#C  0.0  6.0  0.0  9.0
#D  3.1 -3.8  3.3 -6.6
#E  4.1  4.2  4.3  4.4
Related