Trying to export and convert my data to a parquet file. Data is sba data from kaggle that we've transformed bit. Trying to covert it to parquet to load onto a hfds server.
Data link https://www.kaggle.com/mirbektoktogaraev/should-this-loan-be-approved-or-denied
tryin to use the code:
sba.to_parquet('sba.parquet.gzip', compression = 'gzip', partition_cols= 'State')
but get the error:
---------------------------------------------------------------------------
ArrowInvalid Traceback (most recent call last)
<ipython-input-39-377ee6551e44> in <module>
----> 1 sba.to_parquet('sba.parquet.gzip', compression = 'gzip', partition_cols= 'State')
/opt/conda/lib/python3.8/site-packages/pandas/util/_decorators.py in wrapper(*args, **kwargs)
197 else:
198 kwargs[new_arg_name] = new_arg_value
--> 199 return func(*args, **kwargs)
200
201 return cast(F, wrapper)
/opt/conda/lib/python3.8/site-packages/pandas/core/frame.py in to_parquet(self, path, engine, compression, index, partition_cols, storage_options, **kwargs)
2453 from pandas.io.parquet import to_parquet
2454
-> 2455 return to_parquet(
2456 self,
2457 path,
/opt/conda/lib/python3.8/site-packages/pandas/io/parquet.py in to_parquet(df, path, engine, compression, index, storage_options, partition_cols, **kwargs)
388 path_or_buf: FilePathOrBuffer = io.BytesIO() if path is None else path
389
--> 390 impl.write(
391 df,
392 path_or_buf,
/opt/conda/lib/python3.8/site-packages/pandas/io/parquet.py in write(self, df, path, compression, index, storage_options, partition_cols, **kwargs)
150 from_pandas_kwargs["preserve_index"] = index
151
--> 152 table = self.api.Table.from_pandas(df, **from_pandas_kwargs)
153
154 path_or_handle, handles, kwargs["filesystem"] = _get_path_or_handle(
/opt/conda/lib/python3.8/site-packages/pyarrow/table.pxi in pyarrow.lib.Table.from_pandas()
/opt/conda/lib/python3.8/site-packages/pyarrow/pandas_compat.py in dataframe_to_arrays(df, schema, preserve_index, nthreads, columns, safe)
600 for i, maybe_fut in enumerate(arrays):
601 if isinstance(maybe_fut, futures.Future):
--> 602 arrays[i] = maybe_fut.result()
603
604 types = [x.type for x in arrays]
/opt/conda/lib/python3.8/concurrent/futures/_base.py in result(self, timeout)
430 raise CancelledError()
431 elif self._state == FINISHED:
--> 432 return self.__get_result()
433
434 self._condition.wait(timeout)
/opt/conda/lib/python3.8/concurrent/futures/_base.py in __get_result(self)
386 def __get_result(self):
387 if self._exception:
--> 388 raise self._exception
389 else:
390 return self._result
/opt/conda/lib/python3.8/concurrent/futures/thread.py in run(self)
55
56 try:
---> 57 result = self.fn(*self.args, **self.kwargs)
58 except BaseException as exc:
59 self.future.set_exception(exc)
/opt/conda/lib/python3.8/site-packages/pyarrow/pandas_compat.py in convert_column(col, field)
572 e.args += ("Conversion failed for column {!s} with type {!s}"
573 .format(col.name, col.dtype),)
--> 574 raise e
575 if not field_nullable and result.null_count > 0:
576 raise ValueError("Field {} was non-nullable but pandas column "
/opt/conda/lib/python3.8/site-packages/pyarrow/pandas_compat.py in convert_column(col, field)
566
567 try:
--> 568 result = pa.array(col, type=type_, from_pandas=True, safe=safe)
569 except (pa.ArrowInvalid,
570 pa.ArrowNotImplementedError,
/opt/conda/lib/python3.8/site-packages/pyarrow/array.pxi in pyarrow.lib.array()
/opt/conda/lib/python3.8/site-packages/pyarrow/array.pxi in pyarrow.lib._ndarray_to_array()
/opt/conda/lib/python3.8/site-packages/pyarrow/error.pxi in pyarrow.lib.check_status()
ArrowInvalid: ('Could not convert 2004 with type str: tried to convert to int', 'Conversion failed for column ApprovalFY with type object')
Any help would be amazing.