I tried to load csv file into tensor dataset for vertical federated learning. The reference website is https://github.com/OpenMined/PyVertical/blob/master/examples/PyVertical%20Example.ipynb
Below is how I loaded the file but failed
train = pd.read_csv('datatrain.csv') # load data
cols = ["a","b,"c"] # select feature columns
train_feature = train[cols] # create dataset with features
train_target = train['result'] # the dataset with result
# turn them in to torch.tensor data
train_feature_tensor = torch.tensor(train_feature.values)
train_target_tensor = torch.tensor(train_target.values)
# Put them into a TensorDataset
train_tensor = data_utils.dataset.TensorDataset(train_feature_tensor, train_target_tensor)
# them put them in to add_ids()
temp = add_ids(data_utils.dataset.TensorDataset)
temp.data = train_tensor
traindata_ft = temp(train_tensor)
Output:
'TensorDataset' object has no attribute 'size'
They pointed out the problem is from:
assert all(tensors[0].size(0) == tensor.size(0) for tensor in tensors)
which is in:
class TensorDataset(Dataset):
r"""Dataset wrapping tensors.
Each sample will be retrieved by indexing tensors along the first dimension.
Arguments:
*tensors (Tensor): tensors that have the same size of the first dimension.
"""
def __init__(self, *tensors):
assert all(tensors[0].size(0) == tensor.size(0) for tensor in tensors)
self.tensors = tensors
def __getitem__(self, index):
return tuple(tensor[index] for tensor in self.tensors)
def __len__(self):
return self.tensors[0].size(0)
About add_ids(), it is a function that generates unique id to each data row. The original code is below
def add_ids(cls):
"""Decorator to add unique IDs to a dataset
Args:
cls (torch.utils.data.Dataset) : dataset to generate IDs for
Returns:
VerticalDataset : A class which wraps cls to add unique IDs as an attribute,
and returns data, target, id when __getitem__ is called
"""
class VerticalDataset(cls):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.ids = np.array([uuid4() for _ in range(len(self))])
def __getitem__(self, index):
if self.data is None:
img = None
else:
img = self.data[index]
img = Image.fromarray(img.numpy(), mode="L")
if self.transform is not None:
img = self.transform(img)
if self.targets is None:
target = None
else:
target = int(self.targets[index]) if self.targets is not None else None
if self.target_transform is not None:
target = self.target_transform(target)
id = self.ids[index]
# Return a tuple of non-None elements
return (*filter(lambda x: x is not None, (img, target, id)),)
def __len__(self):
if self.data is not None:
return self.data.size(0)
else:
return len(self.targets)
def get_ids(self) -> List[str]:
"""Return a list of the ids of this dataset."""
return [str(id_) for id_ in self.ids]
def sort_by_ids(self):
"""
Sort the dataset by IDs in ascending order
"""
ids = self.get_ids()
sorted_idxs = np.argsort(ids)
if self.data is not None:
self.data = self.data[sorted_idxs]
if self.targets is not None:
self.targets = self.targets[sorted_idxs]
self.ids = self.ids[sorted_idxs]
return VerticalDataset