I'm using:
- Jupyter Lab 3.2.9
- Spark v3.1.2
- Python 3.6.8
- Everything inside VM with Centos
I'm trying to connect to Azure Blob storage:
from pyspark import SparkContext, SparkConf
from pyspark.sql import SparkSession
from pyspark.sql.types import StructType,StructField, StringType, DoubleType
import pyspark
spark = SparkSession.builder.master("local[*]") \
.appName("my_app") \
.config("spark.jars", "<PATH TO THE PROJECT ENV>env/lib/python3.6/site-packages/jars/azure-data-lake-store-sdk-2.2.3.jar,<PATH TO THE PROJECT ENV>/env/lib/python3.6/site-packages/jars/hadoop-azure-3.3.1.jar") \
.getOrCreate()
spark.conf.set("fs.azure.account.auth.type.<INFORMATION>.dfs.core.windows.net", <INFORMATION>)
spark.conf.set("fs.azure.account.oauth.provider.type.<INFORMATION>.dfs.core.windows.net",<INFORMATION>)
spark.conf.set("fs.azure.account.oauth2.client.id.<INFORMATION>.dfs.core.windows.net", <INFORMATION>)
spark.conf.set("fs.azure.account.oauth2.client.secret.<INFORMATION>.dfs.core.windows.net", <INFORMATION>)
spark.conf.set("fs.azure.account.oauth2.client.endpoint.<INFORMATION>.dfs.core.windows.net",<INFORMATION>)
schema = StructType([ \
StructField("Id",StringType(),True), \
StructField("Name",StringType(),True), \
StructField("Country",StringType(),True), \
...
])
data= spark.read\
.format("csv")\
.option("header", True)\
.option("nullValues", "null")\
.schema(schema)\
.load("abfss://<PATH>")
I receive the following error:
Py4JJavaError: An error occurred while calling o46.load. : java.lang.NoClassDefFoundError: org/apache/hadoop/fs/statistics/IOStatisticsSource
at java.lang.ClassLoader.defineClass1(Native Method)
at java.lang.ClassLoader.defineClass(ClassLoader.java:756)
at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142) at java.net.URLClassLoader.defineClass(URLClassLoader.java:473)
at java.net.URLClassLoader.access$100(URLClassLoader.java:74) at java.net.URLClassLoader$1.run(URLClassLoader.java:369)
at java.net.URLClassLoader$1.run(URLClassLoader.java:363)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:362)
at java.lang.ClassLoader.loadClass(ClassLoader.java:418)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352)
at java.lang.ClassLoader.loadClass(ClassLoader.java:351)
at java.lang.ClassLoader.defineClass1(Native Method) at at java.lang.ClassLoader.defineClass(ClassLoader.java:756)
at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142) at java.net.URLClassLoader.defineClass(URLClassLoader.java:473)
at java.net.URLClassLoader.access$100(URLClassLoader.java:74)
at java.net.URLClassLoader$1.run(URLClassLoader.java:369)
at java.net.URLClassLoader$1.run(URLClassLoader.java:363)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:362)
at java.lang.ClassLoader.loadClass(ClassLoader.java:418)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352)
at java.lang.ClassLoader.loadClass(ClassLoader.java:405)
at java.lang.ClassLoader.loadClass(ClassLoader.java:351)
at java.lang.Class.forName0(Native Method)
at java.lang.Class.forName(Class.java:348)
at org.apache.hadoop.conf.Configuration.getClassByNameOrNull(Configuration.java:2532) at org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:2497) at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:2593) at org.apache.hadoop.fs.FileSystem.getFileSystemClass(FileSystem.java:3269) at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3301) at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:124) at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:3352) at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:3320) at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:479)
at org.apache.hadoop.fs.Path.getFileSystem(Path.java:361)
at org.apache.spark.sql.execution.streaming.FileStreamSink$.hasMetadata(FileStreamSink.scala:46) at org.apache.spark.sql.execution.datasources.DataSource.resolveRelation(DataSource.scala:377) at org.apache.spark.sql.DataFrameReader.loadV1Source(DataFrameReader.scala:325) at org.apache.spark.sql.DataFrameReader.$anonfun$load$3(DataFrameReader.scala:307) at scala.Option.getOrElse(Option.scala:189)
at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:307) at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:239) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498)
at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
at py4j.Gateway.invoke(Gateway.java:282)
at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) at py4j.commands.CallCommand.execute(CallCommand.java:79)
at py4j.GatewayConnection.run(GatewayConnection.java:238)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.fs.statistics.IOStatisticsSource
at java.net.URLClassLoader.findClass(URLClassLoader.java:387)
at java.lang.ClassLoader.loadClass(ClassLoader.java:418)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352)
at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ... 55 more
I'm running out of ideas. Anyone can help me?