I am encountering below error message sporadically.
C:\ProgramData\Anaconda3\envs\py39\lib\site-packages\pandas\core\series.py in unstack(self, level, fill_value)
3827 from pandas.core.reshape.reshape import unstack
3828
-> 3829 return unstack(self, level, fill_value)
3830
3831 # ----------------------------------------------------------------------
C:\ProgramData\Anaconda3\envs\py39\lib\site-packages\pandas\core\reshape\reshape.py in unstack(obj, level, fill_value)
428 if is_extension_array_dtype(obj.dtype):
429 return _unstack_extension_series(obj, level, fill_value)
--> 430 unstacker = _Unstacker(
431 obj.index, level=level, constructor=obj._constructor_expanddim
432 )
C:\ProgramData\Anaconda3\envs\py39\lib\site-packages\pandas\core\reshape\reshape.py in __init__(self, index, level, constructor)
116 raise ValueError("Unstacked DataFrame is too big, causing int32 overflow")
117
--> 118 self._make_selectors()
119
120 @cache_readonly
C:\ProgramData\Anaconda3\envs\py39\lib\site-packages\pandas\core\reshape\reshape.py in _make_selectors(self)
150
151 # make the mask
--> 152 remaining_labels = self.sorted_labels[:-1]
153 level_sizes = [len(x) for x in new_levels]
154
pandas\_libs\properties.pyx in pandas._libs.properties.CachedProperty.__get__()
C:\ProgramData\Anaconda3\envs\py39\lib\site-packages\pandas\core\reshape\reshape.py in sorted_labels(self)
137 @cache_readonly
138 def sorted_labels(self):
--> 139 indexer, to_sort = self._indexer_and_to_sort
140 return [line.take(indexer) for line in to_sort]
141
pandas\_libs\properties.pyx in pandas._libs.properties.CachedProperty.__get__()
C:\ProgramData\Anaconda3\envs\py39\lib\site-packages\pandas\core\reshape\reshape.py in _indexer_and_to_sort(self)
127 sizes = [len(x) for x in levs[:v] + levs[v + 1 :] + [levs[v]]]
128
--> 129 comp_index, obs_ids = get_compressed_ids(to_sort, sizes)
130 ngroups = len(obs_ids)
131
C:\ProgramData\Anaconda3\envs\py39\lib\site-packages\pandas\core\sorting.py in get_compressed_ids(labels, sizes)
195 tuple of (comp_ids, obs_group_ids)
196 """
--> 197 ids = get_group_index(labels, sizes, sort=True, xnull=False)
198 return compress_group_index(ids, sort=True)
199
C:\ProgramData\Anaconda3\envs\py39\lib\site-packages\pandas\core\sorting.py in get_group_index(labels, shape, sort, xnull)
139 labels = map(ensure_int64, labels)
140 if not xnull:
--> 141 labels, shape = map(list, zip(*map(maybe_lift, labels, shape)))
142
143 labels = list(labels)
SystemError: \objects\tupleobject.c:914: bad argument to internal function
the above was called from the actual code:
1539 slice_tmp = my_orderlog.groupby(['ticker','slice']).activeOrderSize.sum().to_frame('size').reset_index()
1540 slice_tmp['unit_size'] = slice_tmp['size']/slice_tmp['ticker'].map(wt_map)
1541 logger.info(f'TRYING TO DEBUG: {slice_tmp}')
-> 1542 breakdown = slice_tmp.groupby(['ticker','slice']).unit_size.sum().unstack(level=0)
1543 logger.info(f'TRYING TO DEBUG {breakdown}')
The SystemError of bad argument of internal functions are not repeatable and very difficult (if impossible) to catch. Note that I try to log the in-question dataframe "slice_tmp". but when i get that and run the exact same code, i.e., the "groupby(['ticker','slice']).unit_size.sum().unstack(level=0)" in jupyter notebook, the code runs no problem.
any pointer/ some similar experiences on this odd one please? FWIW i'm running Pandas 1.2.4.