hello everyone I am working on PySpark Python and I have the following code to mock for testing. When I run pytest, I return an error and I think it's failing on show(). The error points to AppData\Local\Programs\Python\Python310\lib\site-packages\py4j\protocol.py:326: Py4JJavaError. Is there an issue with that protocol.py file?
import pytest
from src.components.code import code
from src.shared import *
from tests.helper import *
@pytest.mark.usefixtures("spark_session")
def test_company_code(spark_session: SparkSession):
# Generate mock data
list_df = spark_session.createDataFrame(
[
("a83905a",),
("ghk3945",),
("gdss049",),
("iuoe043",),
("20j48da",),
],
["id"],
)
table_df = spark_session.createDataFrame(
[
("a83905a", "10"),
("ghk3945", "15"),
("gdss049", "100"),
("20j48da", ""),
],
["id", "code"],
)
# Pass mock data to component function
output = code(table_df, list_df)
output.show()
This is the error after running pytest:
answer = 'xro81', gateway_client = <py4j.clientserver.JavaClient object at 0x000002A67F530310>, target_id = 'o80'
name = 'showString'
def get_return_value(answer, gateway_client, target_id=None, name=None):
"""Converts an answer received from the Java gateway into a Python object.
For example, string representation of integers are converted to Python
integer, string representation of objects are converted to JavaObject
instances, etc.
:param answer: the string returned by the Java gateway
:param gateway_client: the gateway client used to communicate with the Java
Gateway. Only necessary if the answer is a reference (e.g., object,
list, map)
:param target_id: the name of the object from which the answer comes from
(e.g., *object1* in `object1.hello()`). Optional.
:param name: the name of the member from which the answer comes from
(e.g., *hello* in `object1.hello()`). Optional.
"""
if is_error(answer)[0]:
if len(answer) > 1:
type = answer[1]
value = OUTPUT_CONVERTER[type](answer[2:], gateway_client)
if answer[1] == REFERENCE_TYPE:
> raise Py4JJavaError(
"An error occurred while calling {0}{1}{2}.\n".
format(target_id, ".", name), value)
E py4j.protocol.Py4JJavaError: An error occurred while calling o80.showString.
E : org.apache.spark.SparkException: Job aborted due to stage failure: Task 6 in stage 0.0 failed 1 times, most recent failure: Lost task 6.0 in stage 0.0 (TID 6) (LTWX5CG1263K36.principalusa.corp.principal.com executor driver): java.io.IOException: Cannot run program "python3": CreateProcess error=2, The system cannot find the file specified
E at java.lang.ProcessBuilder.start(ProcessBuilder.java:1048)
E at org.apache.spark.api.python.PythonWorkerFactory.createSimpleWorker(PythonWorkerFactory.scala:167)
E at org.apache.spark.api.python.PythonWorkerFactory.create(PythonWorkerFactory.scala:109)
E at org.apache.spark.SparkEnv.createPythonWorker(SparkEnv.scala:124)
E at org.apache.spark.api.python.BasePythonRunner.compute(PythonRunner.scala:164)
E at org.apache.spark.api.python.PythonRDD.compute(PythonRDD.scala:65)
E at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:365)
E at org.apache.spark.rdd.RDD.iterator(RDD.scala:329)
E at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
E at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:365)
E at org.apache.spark.rdd.RDD.iterator(RDD.scala:329)
E at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
E at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:365)
E at org.apache.spark.rdd.RDD.iterator(RDD.scala:329)
E at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
E at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:365)
E at org.apache.spark.rdd.RDD.iterator(RDD.scala:329)
E at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
E at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:365)
E at org.apache.spark.rdd.RDD.iterator(RDD.scala:329)
E at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
E at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:365)
E at org.apache.spark.rdd.RDD.iterator(RDD.scala:329)
E at org.apache.spark.shuffle.ShuffleWriteProcessor.write(ShuffleWriteProcessor.scala:59)
E at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:99)
E at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:52)
E at org.apache.spark.scheduler.Task.run(Task.scala:136)
E at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:548)
E at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1504)
E at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:551)
E at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
E at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
E at java.lang.Thread.run(Thread.java:750)
E Caused by: java.io.IOException: CreateProcess error=2, The system cannot find the file specified
E at java.lang.ProcessImpl.create(Native Method)
E at java.lang.ProcessImpl.<init>(ProcessImpl.java:453)
E at java.lang.ProcessImpl.start(ProcessImpl.java:139)
E at java.lang.ProcessBuilder.start(ProcessBuilder.java:1029)
E ... 32 more
E
E Driver stacktrace:
E at org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2672)
E at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2608)
E at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2607)
E at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)
E at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)
E at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)
E at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2607)
E at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:1182)
E at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:1182)
E at scala.Option.foreach(Option.scala:407)
E at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:1182)
E at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2860)
E at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2802)
E at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2791)
E at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49)
E Caused by: java.io.IOException: Cannot run program "python3": CreateProcess error=2, The system cannot find the file specified
E at java.lang.ProcessBuilder.start(ProcessBuilder.java:1048)
E at org.apache.spark.api.python.PythonWorkerFactory.createSimpleWorker(PythonWorkerFactory.scala:167)
E at org.apache.spark.api.python.PythonWorkerFactory.create(PythonWorkerFactory.scala:109)
E at org.apache.spark.SparkEnv.createPythonWorker(SparkEnv.scala:124)
E at org.apache.spark.api.python.BasePythonRunner.compute(PythonRunner.scala:164)
E at org.apache.spark.api.python.PythonRDD.compute(PythonRDD.scala:65)
E at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:365)
E at org.apache.spark.rdd.RDD.iterator(RDD.scala:329)
E at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
E at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:365)
E at org.apache.spark.rdd.RDD.iterator(RDD.scala:329)
E at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
E at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:365)
E at org.apache.spark.rdd.RDD.iterator(RDD.scala:329)
E at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
E at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:365)
E at org.apache.spark.rdd.RDD.iterator(RDD.scala:329)
E at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
E at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:365)
E at org.apache.spark.rdd.RDD.iterator(RDD.scala:329)
E at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
E at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:365)
E at org.apache.spark.rdd.RDD.iterator(RDD.scala:329)
E at org.apache.spark.shuffle.ShuffleWriteProcessor.write(ShuffleWriteProcessor.scala:59)
E at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:99)
E at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:52)
E at org.apache.spark.scheduler.Task.run(Task.scala:136)
E at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:548)
E at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1504)
E at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:551)
E at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
E at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
E at java.lang.Thread.run(Thread.java:750)
E Caused by: java.io.IOException: CreateProcess error=2, The system cannot find the file specified
E at java.lang.ProcessImpl.create(Native Method)
E at java.lang.ProcessImpl.<init>(ProcessImpl.java:453)
E at java.lang.ProcessImpl.start(ProcessImpl.java:139)
E at java.lang.ProcessBuilder.start(ProcessBuilder.java:1029)
E ... 32 more
..\..\AppData\Local\Programs\Python\Python310\lib\site-packages\py4j\protocol.py:326: Py4JJavaError