When I tried to set up SparkContext with SparkConf, I ran into Py4JError. My code is as follows:
from pyspark import SparkContext, SparkConf
conf = SparkConf().setMaster("local").setAppName("Reference")
sc = SparkContext(conf = conf)
And the error is like this:
Py4JError Traceback (most recent call `last)`
<ipython-input-5-313e7d9fd8ee> in <module>()
1 from pyspark import SparkContext, SparkConf
2 conf = SparkConf().setMaster("local").setAppName("Reference")
--> 3 sc = SparkContext(conf = conf)
4 #ref = sc.textFile("reference.csv").map(lambda line: line.split(","))
5 #ref_res = ref.saveAsTextFile("ref.txt")
~/anaconda3/lib/python3.6/site-packages/pyspark/context.py in __init__(self, master, appName, sparkHome, pyFiles, environment, batchSize, serializer, conf, gateway, jsc, profiler_cls)
116 try:
117 self._do_init(master, appName, sparkHome, pyFiles, environment, batchSize, serializer,
--> 118 conf, jsc, profiler_cls)
119 except:
120 # If an error occurs, clean up in order to allow future SparkContext creation:
~/anaconda3/lib/python3.6/site-packages/pyspark/context.py in _do_init(self, master, appName, sparkHome, pyFiles, environment, batchSize, serializer, conf, jsc, profiler_cls)
193 # data via a socket.
194 # scala's mangled names w/ $ in them require special treatment.
--> 195 self._encryption_enabled = self._jvm.PythonUtils.getEncryptionEnabled(self._jsc)
196
197 self.pythonExec = os.environ.get("PYSPARK_PYTHON", 'python')
~/anaconda3/lib/python3.6/site-packages/py4j/java_gateway.py in __getattr__(self, name)
1514 else:
1515 raise Py4JError(
-> 1516 "{0}.{1} does not exist in the JVM".format(self._fqn, name))
1517
1518 def _get_args(self, args):
Py4JError: org.apache.spark.api.python.PythonUtils.getEncryptionEnabled does not exist in the JVM
Any help or advice please.