Python newbie and first time poster to StackOverflow, so please go easy on me :)
I am using pandas-dedupe library to assist with fuzzy matching/ clustering a df 8 columns in total (from excel)
I was able to successfully label 20 odd rows, however at the clustering stage I got the error:
ValueError: Iteration of zero-sized operands is not enabled
There is practically 0 info online for this, so said I would reach out here in case anyone would know
Thanks in advance
EDIT: See full code and stack trace below
import pandas as pd
import pandas_dedupe
df = pd.read_excel('apr_1.xlsx')
df1 = df[['IND_FORENAME', 'IND_SURNAME', 'IND_BIRTH_DT', 'IND_ADDR1', 'IND_MOBILE', 'IND_HOME_PHONE', 'MAIN_MEM_FORENAME', 'MAIN_MEM_SURNAME']]
df1 = pandas_dedupe.dedupe_dataframe(df1, ['IND_FORENAME', 'IND_SURNAME', 'IND_BIRTH_DT', 'IND_ADDR1', 'IND_MOBILE', 'IND_HOME_PHONE', 'MAIN_MEM_FORENAME', 'MAIN_MEM_SURNAME'])
First a warning comes up
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead
Then the stack trace reads:
ValueError Traceback (most recent call last)
Cell In [12], line 1
----> 1 df1 = pandas_dedupe.dedupe_dataframe(df1, ['IND_FORENAME', 'IND_SURNAME', 'IND_BIRTH_DT', 'IND_ADDR1', 'IND_MOBILE', 'IND_HOME_PHONE', 'MAIN_MEM_FORENAME', 'MAIN_MEM_SURNAME'])
File ~/fuzzyMatchingPOC/fuzzyMatching/lib/python3.8/site-packages/pandas_dedupe/dedupe_dataframe.py:252, in dedupe_dataframe(df, field_properties, canonicalize, config_name, update_model, threshold, sample_size, n_cores)
248 deduper = _train(settings_file, training_file, data_d, field_properties,
249 sample_size, update_model, n_cores)
251 # Cluster the records
--> 252 clustered_df = _cluster(deduper, data_d, threshold, canonicalize)
253 results = df.join(clustered_df, how='left')
254 results.drop(['dictionary'], axis=1, inplace=True)
File ~/fuzzyMatchingPOC/fuzzyMatching/lib/python3.8/site-packages/pandas_dedupe/dedupe_dataframe.py:146, in _cluster(deduper, data, threshold, canonicalize)
144 # ## Clustering
145 print('Clustering...')
--> 146 clustered_dupes = deduper.partition(data, threshold)
148 print('# duplicate sets', len(clustered_dupes))
150 # Convert data_d to string so that Price & LatLong won't get traceback
151 # during dedupe.canonicalize()
File ~/fuzzyMatchingPOC/fuzzyMatching/lib64/python3.8/site-packages/dedupe/api.py:177, in DedupeMatching.partition(self, data, threshold)
175 clusters = self.cluster(pair_scores, threshold)
176 clusters = self._add_singletons(data, clusters)
--> 177 clusters = list(clusters)
178 _cleanup_scores(pair_scores)
179 return clusters
File ~/fuzzyMatchingPOC/fuzzyMatching/lib64/python3.8/site-packages/dedupe/api.py:185, in DedupeMatching._add_singletons(self, data, clusters)
181 def _add_singletons(self, data: Data, clusters: Clusters) -> Clusters:
183 singletons = set(data.keys())
--> 185 for record_ids, score in clusters:
186 singletons.difference_update(record_ids)
187 yield (record_ids, score)
File ~/fuzzyMatchingPOC/fuzzyMatching/lib64/python3.8/site-packages/dedupe/api.py:334, in DedupeMatching.cluster(self, scores, threshold)
272 r"""From the similarity scores of pairs of records, decide which groups
273 of records are all referring to the same entity.
274
(...)
329
330 """
332 logger.debug("matching done, begin clustering")
--> 334 yield from clustering.cluster(scores, threshold)
File ~/fuzzyMatchingPOC/fuzzyMatching/lib64/python3.8/site-packages/dedupe/clustering.py:238, in cluster(dupes, threshold, max_components)
235 distance_threshold = 1 - threshold
236 dupe_sub_graphs = connected_components(dupes, max_components)
--> 238 for sub_graph in dupe_sub_graphs:
239 if len(sub_graph) > 1:
241 i_to_id, condensed_distances, N = condensedDistance(sub_graph)
File ~/fuzzyMatchingPOC/fuzzyMatching/lib64/python3.8/site-packages/dedupe/clustering.py:51, in connected_components(edgelist, max_components)
48 else:
49 copy_to_mmap_record_array(unlabeled_edgelist, edgelist, ["pairs", "score"])
---> 51 yield from _connected_components(edgelist, max_components)
53 edgelist._mmap.close()
File ~/fuzzyMatchingPOC/fuzzyMatching/lib64/python3.8/site-packages/dedupe/clustering.py:99, in _connected_components(edgelist, max_components)
96 cut_point = numpy.searchsorted(sub_graph["score"], threshold)
97 filtered_sub_graph = sub_graph[max(cut_point, 2) :]
---> 99 for sub_graph in _connected_components(filtered_sub_graph, max_components):
100 yield sub_graph[["pairs", "score"]]
101 else:
File ~/fuzzyMatchingPOC/fuzzyMatching/lib64/python3.8/site-packages/dedupe/clustering.py:59, in _connected_components(edgelist, max_components)
56 def _connected_components(
57 edgelist: Scores, max_components: int
58 ) -> Generator[Scores, None, None]:
---> 59 component_stops = union_find(edgelist)
61 start = 0
62 for stop in component_stops:
File ~/fuzzyMatchingPOC/fuzzyMatching/lib64/python3.8/site-packages/dedupe/clustering.py:114, in union_find(scored_pairs)
111 edgelist = scored_pairs["pairs"]
112 labels = scored_pairs["label"]
--> 114 it = numpy.nditer(edgelist, ["external_loop"])
116 n_edges = len(scored_pairs)
118 array_type = "H"
ValueError: Iteration of zero-sized operands is not enabled