The situation you describe seems to be more suitable for a generator that fetches the next integer, or the next row from the file and allows you to process that.
def sanify(s):
while s.startswith('['):
s = s[1:]
while s.endswith(']'):
s = s[:-1]
return int(s)
def get_numbers(file_obj):
file_obj.seek(0)
i = j = 0
for line in file_obj:
for item in line.split(', '):
if item and not item.isspace():
yield sanify(item), i, j
j += 1
i += 1
j = 0
This ensures only one line at a time ever resides in memory.
This can be used like:
import io
s = '''[[2, 2, 6, 10, 2, 6, 7, 15, 14, 10, 17, 14, 7, 14, 15, 7, 17],
[3, 3, 7, 11, 3, 7, 0, 11, 7, 16, 0, 17, 17, 7, 16, 0, 0],
[4, 4, 8, 7, 4, 13, 0, 0, 15, 7, 8, 7, 0, 7, 0, 15, 13],
[5, 5, 9, 12, 5, 14, 7, 13, 9, 14, 16, 12, 13, 14, 7, 16, 7]]'''
items = get_numbers(io.StringIO(s))
for item, i, j in items:
print(item, i, j)
If you really want to be able to access an arbitrary element of the matrix, you could adapt the above logic into a class implementing __getitem__ and you would only need to keep track of the position of the beginning of each line.
In code, this would look like:
class MatrixData(object):
def __init__(self, file_obj):
self._file_obj = file_obj
self._line_offsets = list(self._get_line_offsets(file_obj))[:-1]
file_obj.seek(0)
row = list(self._read_row(file_obj.readline()))
self.shape = len(self._line_offsets), len(row)
self.length = self.shape[0] * self.shape[1]
def __len__(self):
return self.length
def __iter__(self):
self._file_obj.seek(0)
i = j = 0
for line in self._file_obj:
for item in _read_row(line):
yield item, i, j
j += 1
i += 1
j = 0
def __getitem__(self, indices):
i, j = indices
self._file_obj.seek(self._line_offsets[i])
line = self._file_obj.readline()
row = self._read_row(line)
return row[j]
@staticmethod
def _get_line_offsets(file_obj):
file_obj.seek(0)
yield file_obj.tell()
for line in file_obj:
yield file_obj.tell()
@staticmethod
def _read_row(line):
for item in line.split(', '):
if item and not item.isspace():
yield MatrixData._sanify(item)
@staticmethod
def _sanify(item, dtype=int):
while item.startswith('['):
item = item[1:]
while item.endswith(']'):
item = item[:-1]
return dtype(item)
class MatrixData(object):
def __init__(self, file_obj):
self._file_obj = file_obj
self._line_offsets = list(self._get_line_offsets(file_obj))[:-1]
file_obj.seek(0)
row = list(self._read_row(file_obj.readline()))
self.shape = len(self._line_offsets), len(row)
self.length = self.shape[0] * self.shape[1]
def __len__(self):
return self.length
def __iter__(self):
self._file_obj.seek(0)
i = j = 0
for line in self._file_obj:
for item in self._read_row(line):
yield item, i, j
j += 1
i += 1
j = 0
def __getitem__(self, indices):
i, j = indices
self._file_obj.seek(self._line_offsets[i])
line = self._file_obj.readline()
row = list(self._read_row(line))
return row[j]
@staticmethod
def _get_line_offsets(file_obj):
file_obj.seek(0)
yield file_obj.tell()
for line in file_obj:
yield file_obj.tell()
@staticmethod
def _read_row(line):
for item in line.split(', '):
if item and not item.isspace():
yield MatrixData._sanify(item)
@staticmethod
def _sanify(item, dtype=int):
while item.startswith('['):
item = item[1:]
while item.endswith(']'):
item = item[:-1]
return dtype(item)
To be used as:
m = MatrixData(io.StringIO(s))
# get total number of elements
len(m)
# get number of row and col
m.shape
# access a specific element
m[3, 12]
# iterate through
for x, i, j in m:
...
