Scipy and Sklearn Yeo-Johnson normalization results do not match

user_915783 0
Viewed 398

I was running Yeo Johnson Transform and followed the example given on Scipy website. Scipy link I also compared it with Sklearn implementation. here is the code: i

    import seaborn as sns
    from sklearn.preprocessing import PowerTransformer
    from scipy import stats
    import matplotlib.pyplot as plt
    import numpy as np

    fig = plt.figure( figsize=(10,10))
    ax1 = fig.add_subplot(421)
    x = stats.loggamma.rvs(5, size=500) + 5
    prob = stats.probplot(x, dist=stats.norm, plot=ax1)
    ax1.set_xlabel('')
    ax1.set_title('Probplot')

    ax2 = fig.add_subplot(422)
    sns.distplot(x, color="skyblue")
    ax2.set_title('Distribution of Data')

    ax3 = fig.add_subplot(423)
    xt_scipy, lmbda = stats.yeojohnson(x)
    prob = stats.probplot(xt_scipy, dist=stats.norm, plot=ax3)
    ax3.set_xlabel('')
    ax3.set_title('Probplot:Yeo-Johnson:Scipy')

    ax4 = fig.add_subplot(424)
    sns.distplot(xt_scipy, color="skyblue")
    ax4.set_title('Distribution of Transformed Data')

    ax5 = fig.add_subplot(425)
    pt = PowerTransformer(method = 'yeo-johnson',standardize = True)
    xt_sklearn = pt.fit_transform(x.reshape(-1,1))
    prob = stats.probplot(xt_sklearn.flatten(), dist=stats.norm, plot=ax5)
    ax5.set_xlabel('')
    ax5.set_title('Probplot:Yeo-Johnson:Sklearn')

    ax6 = fig.add_subplot(426)
    sns.distplot(xt_sklearn, color="skyblue")
    ax6.set_title('Distribution of Transformed Data')
    plt.tight_layout(h_pad=0.9, w_pad=0.9)
    plt.show()

Looking at the attached figure, it can be seen that both methods seem to normalize data as expected , evident from the quantiles.
But the distribution plots of transformed data, for both libraries , although of same shape, has different range of values. Why are the transformed values different? which one corresponds to the true Yeo Johnson formula? Figure1 sedy

scipy normalization python
1 Answers

It was my mistake. I didn't realize Sklearn does standard scale after Power transform by default. Here is modification of the code that creates matching results.

    import seaborn as sns
    import sklearn.preprocessing
    from sklearn.preprocessing import PowerTransformer, StandardScaler
    from scipy import stats
    import matplotlib.pyplot as plt
    import numpy as np

    ss = StandardScaler()
    fig = plt.figure( figsize=(10,10))
    ax1 = fig.add_subplot(441)
    x = stats.loggamma.rvs(5, size=500) + 5
    prob = stats.probplot(x, dist=stats.norm, plot=ax1)
    ax1.set_xlabel('')
    ax1.set_title('Probplot')

    ax2 = fig.add_subplot(442)
    sns.distplot(x, color="skyblue")
    ax2.set_title('Distribution of Data')

    ax5 = fig.add_subplot(445)
    xt_scipy, lmbda = stats.yeojohnson(x)
    prob = stats.probplot(xt_scipy, dist=stats.norm, plot=ax5)
    ax5.set_xlabel('')
    ax5.set_title('Probplot:Yeo-Johnson:Scipy')

    ax6 = fig.add_subplot(446)
    sns.distplot(xt_scipy, color="skyblue")
    ax6.set_title('Distribution of Transformed Data')

    ax7 = fig.add_subplot(447)
    xt_scipy_ss, lmbda = stats.yeojohnson(x)
    xt_scipy_ss = ss.fit_transform(xt_scipy_ss.reshape(-1, 1))
    prob = stats.probplot(xt_scipy_ss.flatten(), dist=stats.norm, plot=ax7)
    ax7.set_xlabel('')
    ax7.set_title('Probplot:Yeo-Johnson + Stand Scal:Scipy')

    ax8 = fig.add_subplot(448)
    sns.distplot(xt_scipy_ss, color="skyblue")
    ax8.set_title('Distribution of Transformed Data')

    ax9 = fig.add_subplot(449)
    pt = PowerTransformer(method = 'yeo-johnson',standardize = False)
    xt_sklearn = pt.fit_transform(x.reshape(-1,1))
    prob = stats.probplot(xt_sklearn.flatten(), dist=stats.norm, plot=ax9)
    ax9.set_xlabel('')
    ax9.set_title('Probplot:Yeo-Johnson:Sklearn')

    ax10 = fig.add_subplot(4,4,10)
    sns.distplot(xt_sklearn, color="skyblue")
    ax10.set_title('Distribution of Transformed Data')

    ax11 = fig.add_subplot(4,4,11)
    pt = PowerTransformer(method='yeo-johnson', standardize=True)
    xt_sklearn_ss = pt.fit_transform(x.reshape(-1, 1))
    prob = stats.probplot(xt_sklearn_ss.flatten(), dist=stats.norm, plot=ax11)
    ax11.set_xlabel('')
    ax11.set_title('Probplot:Yeo-Johnson:Sklearn with Stand Scal')

    ax12 = fig.add_subplot(4, 4, 12)
    sns.distplot(xt_sklearn_ss, color="skyblue")
    ax12.set_title('Distribution of Transformed Data')
    plt.tight_layout(h_pad=0.9, w_pad=0.9)
    plt.show()
user_9157830
Related