Is there a way to find which variables an object is stored in in python?

Question

I have some object "Obj" that is stored in multiple variables across my program. Is there some function that can find where it is stored

>>>class blank:
...    pass
...
>>>Obj=blank()
>>>a=Obj
>>>b=a
>>>c=[1, 3, Obj]
>>>## desired output
>>>Obj.mystery_function()
["a", "b", "c[2]"]

Thank you in advance for your answers

Lenormju · Answer

Literal answer to "how to find where my data is stored" :

class Blank:
    pass


obj = Blank()
a = obj
b = a
c = [1, 3, obj]


import gc

def find_all_transitive_references_to(something):
    something_id = id(something)
    for referrer in gc.get_referrers(something):
        if isinstance(referrer, dict):
            # find the corresponding key(s)
            keys = set(key for key, value in referrer.items() if id(value) == something_id)
            this_function_local_scope = id(locals())
            if id(referrer) == this_function_local_scope:
                pass  # redundant
            elif '__name__' in referrer.keys():  # assume it is a module
                print(f"module {referrer['__name__']}:", keys)
            else:
                print("dict keys:", keys)  # TODO
        else:
            print("else:", type(referrer), referrer)  # TODO


find_all_transitive_references_to(obj)

else:  [1, 3, <__main__.Blank object at 0x7fca0fb9be50>]
module __main__: {'b', 'obj', 'a'}

That's a good start. Most of the work is done by gc.get_refferers which does half the work. But as it is intended to be used by a garbage collector, it does not produce a clean and nice output.

But going farther gets immensely difficult, because you have to recursively explore referrers up the refs chains. And barring cycles and joints in the ref graph, you just can't easily guess a name.
Here is what I got so far :

class Blank:
    pass


obj = Blank()
a = obj
b = a
c = [1, 3, obj]
d = {'1': obj, '2': 'foo'}
e = {obj: '1', '2': 'foo'}
f = {'1': obj, '2': obj}
g = {obj}

def function_with_default_mutable_argument(o=obj):
    pass


import dataclasses
import enum
import gc


class ReferenceSource(enum.Enum):
    MODULE_VAR = enum.auto
    LIST_ELEM = enum.auto
    TUPLE_ELEM = enum.auto
    DICT_KEY = enum.auto
    DICT_VAL = enum.auto
    UNKNOWN = enum.auto


@dataclasses.dataclass
class Reference:
    python_name: str
    display_name: str
    source: ReferenceSource


# def get


def find_all_transitive_references_to(something, excludes=None):
    if excludes is None:
        excludes = []
    something_id = id(something)
    results = []
    for referrer in gc.get_referrers(something):
        if referrer in excludes:
            continue
        if isinstance(referrer, dict):
            # find the corresponding key(s)
            keys = set(key for key, value in referrer.items() if id(value) == something_id)
            this_function_local_scope = id(locals())
            if id(referrer) == this_function_local_scope:
                pass  # redundant
            elif '__name__' in referrer.keys():  # assume it is a module  # FIXME: functions scopes are alike
                for key in keys:
                    if key == 'something':
                        continue
                    results.append(Reference(f"{referrer['__name__']!s}.{key!s}", f"module {referrer['__name__']} variable {key!r}", ReferenceSource.MODULE_VAR))
            else:
                for key, value in referrer.items():
                    if id(key) == something_id:
                        results.append(Reference(f"TODO:NAME key", f"TODO:NAME dict key", ReferenceSource.DICT_KEY))
                    elif id(value) == something_id:
                        results.append(Reference(f"TODO:NAME [{key!r}]", f"TODO:NAME dict [{key!r}]", ReferenceSource.DICT_VAL))
        elif isinstance(referrer, list):
            for index in (index for index, value in enumerate(referrer) if id(value) == something_id):
                results.append(Reference(f"TODO:NAME [{index!s}]", f"TODO:NAME list [{index!s}]", ReferenceSource.LIST_ELEM))
        elif isinstance(referrer, tuple):
            # may be a function parameter default value
            for index in (index for index, value in enumerate(referrer) if id(value) == something_id):
                results.append(Reference(f"TODO:NAME [{index!s}]", f"TODO:NAME tuple [{index!s}]", ReferenceSource.TUPLE_ELEM))
        elif isinstance(referrer, set):
            referrer_referrers = find_all_transitive_references_to(referrer, excludes=[id(locals())])
            print("ref_ref count=", len(referrer_referrers), str(referrer_referrers))
            module_referrer_referrer = next((ref_ref for ref_ref in referrer_referrers if ref_ref.source is ReferenceSource.MODULE_VAR), None)
            parent_name = module_referrer_referrer.python_name if module_referrer_referrer is not None else "?"
            count = sum(1 for value in referrer if id(value) == something_id)
            results.append(Reference(f"{parent_name!s} ({count!s} times)", f"TODO:NAME ({count!s} times)", ReferenceSource.TUPLE_ELEM))
        else:
            results.append(Reference(f"?", f"TODO:unknown", ReferenceSource.UNKNOWN))
    return results

print("
".join(map(str, find_all_transitive_references_to(obj))))

ref_ref count= 3 [Reference(python_name='TODO:NAME [4]', display_name='TODO:NAME list [4]', source=>), Reference(python_name="TODO:NAME ['referrer']", display_name="TODO:NAME dict ['referrer']", source=>), Reference(python_name='__main__.g', display_name="module __main__ variable 'g'", source=>)]
Reference(python_name='TODO:NAME [2]', display_name='TODO:NAME list [2]', source=>)
Reference(python_name="TODO:NAME ['1']", display_name="TODO:NAME dict ['1']", source=>)
Reference(python_name='TODO:NAME key', display_name='TODO:NAME dict key', source=>)
Reference(python_name="TODO:NAME ['1']", display_name="TODO:NAME dict ['1']", source=>)
Reference(python_name="TODO:NAME ['2']", display_name="TODO:NAME dict ['2']", source=>)
Reference(python_name='TODO:NAME [4] (1 times)', display_name='TODO:NAME (1 times)', source=>)
Reference(python_name='TODO:NAME [0]', display_name='TODO:NAME tuple [0]', source=>)
Reference(python_name='__main__.a', display_name="module __main__ variable 'a'", source=>)
Reference(python_name='__main__.obj', display_name="module __main__ variable 'obj'", source=>)
Reference(python_name='__main__.b', display_name="module __main__ variable 'b'", source=>)

But it gets very hairy because gc is not a very good tool for the problem. I suggest you listen to the suggestion by @Marat and explain why you would need it. There must a be a MUCH MUCH simpler way to achieve what you actually want.

Is there a way to find which variables an object is stored in in python?

1 Answers