I am a bit confused about the internal workings of pandas groupby transform functionality. I will use the following example to explain the areas of confusion:
df = pd.DataFrame({
"group": ['a', 'a', 'a', 'b', 'b', 'b', 'c', 'c', 'c'],
"name": ["A1", "B1", "C1", "A2", "B2", "C2", "A3", "B3", "C3"],
"score": [97, 93, 92, 94, 95, 93, 100, 92, 93],
"p": [1,2,3,4,5,6,7,8,9]
})
df.index=[3,5,7,9,11,13,15,17,19]
g = df.groupby('group')
Confusions: How the Data actually gets iterated over and how does transform decide the columns on which to apply the transformation?
- If I do no operations and pass just a print function to the transform, then the iteration is first over the columns of the first group one by one, then it passes the entire first group, then it is over the columns in the second group one by one and then the entire second group and so on:
g.transform(lambda x: print('\n'*2,x,'*'*20))
prints that show up:
3 A1
5 B1
7 C1
Name: name, dtype: object ********************
3 97
5 93
7 92
Name: score, dtype: int64 ********************
3 1
5 2
7 3
Name: p, dtype: int64 ********************
name score p
3 A1 97 1
5 B1 93 2
7 C1 92 3 ********************
9 A2
11 B2
13 C2
Name: name, dtype: object ********************
9 94
11 95
13 93
Name: score, dtype: int64 ********************
9 4
11 5
13 6
Name: p, dtype: int64 ********************
name score p
9 A2 94 4
11 B2 95 5
13 C2 93 6 ********************
15 A3
17 B3
19 C3
Name: name, dtype: object ********************
15 100
17 92
19 93
Name: score, dtype: int64 ********************
15 7
17 8
19 9
Name: p, dtype: int64 ********************
name score p
15 A3 100 7
17 B3 92 8
19 C3 93 9 ********************
- If I make any operations in the transform function that can be applied to both the object types and the numeric types, then the iteration behaviour persists. Each column in the 1st group, the entire first group, each column in the second group, the second group and so on.
g.transform(lambda x: print('\n'*2,x*2,'*'*20))
3 A1A1
5 B1B1
7 C1C1
Name: name, dtype: object ********************
3 194
5 186
7 184
Name: score, dtype: int64 ********************
3 2
5 4
7 6
Name: p, dtype: int64 ********************
name score p
3 A1A1 194 2
5 B1B1 186 4
7 C1C1 184 6 ********************
9 A2A2
11 B2B2
13 C2C2
Name: name, dtype: object ********************
9 188
11 190
13 186
Name: score, dtype: int64 ********************
9 8
11 10
13 12
Name: p, dtype: int64 ********************
name score p
9 A2A2 188 8
11 B2B2 190 10
13 C2C2 186 12 ********************
15 A3A3
17 B3B3
19 C3C3
Name: name, dtype: object ********************
15 200
17 184
19 186
Name: score, dtype: int64 ********************
15 14
17 16
19 18
Name: p, dtype: int64 ********************
name score p
15 A3A3 200 14
17 B3B3 184 16
19 C3C3 186 18 ********************
- However, if I make any numeric-only computations in the transform function, then the entire group after each column iteration is not iterated over. The order of iteration is also changed. It is all the groups in the first columns, then all the groups in the second columns, then the third column and so on and so forth. Earlier it was the first group of the first column, then the first group of the second column etc and then move similarly to the next group after exhausting all the columns in the first group. Transform is also smart enough to understand that it is a numeric computation and to handle the object typed columns gracefully without throwing errors. It simply skips the object typed columns:
g.transform(lambda x: print('\n'*2,x-x.mean(),'\n',x,'*'*20))
prints that show up:
3 3.0
5 -1.0
7 -2.0
Name: a, dtype: float64
3 97
5 93
7 92
Name: a, dtype: int64 ********************
9 0.0
11 1.0
13 -1.0
Name: b, dtype: float64
9 94
11 95
13 93
Name: b, dtype: int64 ********************
15 5.0
17 -3.0
19 -2.0
Name: c, dtype: float64
15 100
17 92
19 93
Name: c, dtype: int64 ********************
3 -1.0
5 0.0
7 1.0
Name: a, dtype: float64
3 1
5 2
7 3
Name: a, dtype: int64 ********************
9 -1.0
11 0.0
13 1.0
Name: b, dtype: float64
9 4
11 5
13 6
Name: b, dtype: int64 ********************
15 -1.0
17 0.0
19 1.0
Name: c, dtype: float64
15 7
17 8
19 9
Name: c, dtype: int64 ********************
- Yet, if I make any object-only computations in the transform function, then the Transform flounders in handling this case gracefully and throws an error when the iteration reached a numeric column. Also, here it did not go through a column entirely before moving on to the next column. It decides to exhaust all the columns in the first group and then move on to the next group:
g.transform(lambda x: print('\n'*2,x.str.cat(x),'*'*20))
output:
3 A1A1
5 B1B1
7 C1C1
Name: name, dtype: object ********************
---------------------------------------------------------------------------
AttributeError Traceback (most recent call last)
/tmp/ipykernel_2705229/621028603.py in <module>
----> 1 g.transform(lambda x: print('\n'*2,x.str.cat(x),'*'*20))
~/miniconda3/envs/pyspark/lib/python3.9/site-packages/pandas/core/groupby/generic.py in transform(self, func, engine, engine_kwargs, *args, **kwargs)
1357
1358 if not isinstance(func, str):
-> 1359 return self._transform_general(func, *args, **kwargs)
1360
1361 elif func not in base.transform_kernel_allowlist:
~/miniconda3/envs/pyspark/lib/python3.9/site-packages/pandas/core/groupby/generic.py in _transform_general(self, func, *args, **kwargs)
1304 # Try slow path and fast path.
1305 try:
-> 1306 path, res = self._choose_path(fast_path, slow_path, group)
1307 except TypeError:
1308 return self._transform_item_by_item(obj, fast_path)
~/miniconda3/envs/pyspark/lib/python3.9/site-packages/pandas/core/groupby/generic.py in _choose_path(self, fast_path, slow_path, group)
1415 def _choose_path(self, fast_path: Callable, slow_path: Callable, group: DataFrame):
1416 path = slow_path
-> 1417 res = slow_path(group)
1418
1419 # if we make it here, test if we can use the fast path
~/miniconda3/envs/pyspark/lib/python3.9/site-packages/pandas/core/groupby/generic.py in <lambda>(group)
1408 else:
1409 fast_path = lambda group: func(group, *args, **kwargs)
-> 1410 slow_path = lambda group: group.apply(
1411 lambda x: func(x, *args, **kwargs), axis=self.axis
1412 )
~/miniconda3/envs/pyspark/lib/python3.9/site-packages/pandas/core/frame.py in apply(self, func, axis, raw, result_type, args, **kwds)
7766 kwds=kwds,
7767 )
-> 7768 return op.get_result()
7769
7770 def applymap(self, func, na_action: Optional[str] = None) -> DataFrame:
~/miniconda3/envs/pyspark/lib/python3.9/site-packages/pandas/core/apply.py in get_result(self)
183 return self.apply_raw()
184
--> 185 return self.apply_standard()
186
187 def apply_empty_result(self):
~/miniconda3/envs/pyspark/lib/python3.9/site-packages/pandas/core/apply.py in apply_standard(self)
274
275 def apply_standard(self):
--> 276 results, res_index = self.apply_series_generator()
277
278 # wrap results
~/miniconda3/envs/pyspark/lib/python3.9/site-packages/pandas/core/apply.py in apply_series_generator(self)
288 for i, v in enumerate(series_gen):
289 # ignore SettingWithCopy here in case the user mutates
--> 290 results[i] = self.f(v)
291 if isinstance(results[i], ABCSeries):
292 # If we have a view on v, we need to make a copy because
~/miniconda3/envs/pyspark/lib/python3.9/site-packages/pandas/core/groupby/generic.py in <lambda>(x)
1409 fast_path = lambda group: func(group, *args, **kwargs)
1410 slow_path = lambda group: group.apply(
-> 1411 lambda x: func(x, *args, **kwargs), axis=self.axis
1412 )
1413 return fast_path, slow_path
/tmp/ipykernel_2705229/621028603.py in <lambda>(x)
----> 1 g.transform(lambda x: print('\n'*2,x.str.cat(x),'*'*20))
~/miniconda3/envs/pyspark/lib/python3.9/site-packages/pandas/core/generic.py in __getattr__(self, name)
5459 or name in self._accessors
5460 ):
-> 5461 return object.__getattribute__(self, name)
5462 else:
5463 if self._info_axis._can_hold_identifiers_and_holds_name(name):
~/miniconda3/envs/pyspark/lib/python3.9/site-packages/pandas/core/accessor.py in __get__(self, obj, cls)
178 # we're accessing the attribute of the class, i.e., Dataset.geo
179 return self._accessor
--> 180 accessor_obj = self._accessor(obj)
181 # Replace the property with the accessor object. Inspired by:
182 # https://www.pydanny.com/cached-property.html
~/miniconda3/envs/pyspark/lib/python3.9/site-packages/pandas/core/strings/accessor.py in __init__(self, data)
152 from pandas.core.arrays.string_ import StringDtype
153
--> 154 self._inferred_dtype = self._validate(data)
155 self._is_categorical = is_categorical_dtype(data.dtype)
156 self._is_string = isinstance(data.dtype, StringDtype)
~/miniconda3/envs/pyspark/lib/python3.9/site-packages/pandas/core/strings/accessor.py in _validate(data)
215
216 if inferred_dtype not in allowed_types:
--> 217 raise AttributeError("Can only use .str accessor with string values!")
218 return inferred_dtype
219
AttributeError: Can only use .str accessor with string values!
I am not able to form a cohesive understanding of how the transform works in pandas. Can you please help me understand what the under-the-hood universe of pandas groupby transform functionality is?