Custom StandardScaler for each column

I'm trying to build a custom transformer for a sklearn pipeline using StandardScaler but first removing each column's median value. These operations are column-specific but I'm not sure what the most efficient way to do this is.

This is the code I have so far. It works but it seems overly complicated. Surely there's a way to do this without looping through all the columns?

class Standardize(BaseEstimator, TransformerMixin):
    def __init__(self):
        super().__init__()
        self.medians = pd.Series()
        self.scalers = {}

    def fit(self, X, y=None):
        self.medians = X.median(skipna=True)
        for col in X:
            median_value = self.medians.loc[col]
            scaler = StandardScaler()
            scaler.fit(X.loc[X[col] != median_value, col].to_frame())
            self.scalers[col] = scaler
        return self

    def transform(self, X, y=None):
        scaled = {}
        for col in X:
            median_value = self.medians.loc[col]
            scaler = self.scalers[col]
            scaled[col] = scaler.transform(X[[col]]).reshape(-1, )
        return pd.DataFrame.from_dict(scaled, orient='columns')