I have written a code. It first get all the files in /FileStore/tables, then it looks for three columns of "A", "B", "G". when I try similar code in my jupytor notebook, it works fine. But here in Databricks in the line ax.scatter3D(x,y,-z), I get this error:
TypeError: bad operand type for unary -: 'str'
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<command-4099615870847045> in <module>
25
26 #Creating plot
---> 27 ax.scatter3D(x,y,-z)
28
29 # Creating legend and Labels
/databricks/python/lib/python3.8/site-packages/pandas/core/generic.py in __neg__(self)
1405 or is_object_dtype(values)
1406 ):
-> 1407 arr = operator.neg(values)
1408 else:
1409 raise TypeError(f"Unary negative expects numeric dtype, not {values.dtype}")
TypeError: bad operand type for unary -: 'str'
and here is my code:
This function reads the files in directory:
def get_csv_files(directory_path):
"""recursively list path of all csv files in path directory """
csv_files = []
files_to_treat = dbutils.fs.ls(directory_path)
while files_to_treat:
path = files_to_treat.pop(0).path
if path.endswith('/'):
files_to_treat += dbutils.fs.ls(path)
elif path.endswith('.csv'):
csv_files.append(path)
return csv_files
and this is the rest of code
import pandas as pd
import numpy as np
from mpl_toolkits import mplot3d
import matplotlib.pyplot as plt
files = get_csv_files('/FileStore/tables/')
# Creating figure
fig = plt.figure(figsize = (15, 15))
ax = plt.axes(projection ="3d")
for i in files:
df = spark.read.format('csv').options(header='true').load(i).toPandas()
if "A(m)" and "B(m)" and "G(m)" in df.columns:
x = df.loc[:,["A(m)"]]
y = df.loc[:,["B(m)"]]
z = df.loc[:,["G(m)"]]
elif "A" and "B" and "G" in df.columns:
x = df.loc[:,["A"]]
y = df.loc[:,["B"]]
z = df.loc[:,["G"]]
#Creating plot
ax.scatter3D(x,y,-z)
# Creating legend and Labels
plt.xlabel("A")
plt.ylabel("B")
# show plot
plt.show()