I am learning Bunsen https://engineering.cerner.com/bunsen/0.5.10-SNAPSHOT/index.html for handling FHIR data with Pyspark
the folder bundles contain a fhir bundle which is a collection of resources as json objects
I downloaded all the requirements needed such as the bunsen module and the jar file for pyspark and I run these commands:
>>> import pyspark
>>> from bunsen.r4.bundles import load_from_directory, extract_entry
>>> bundles = load_from_directory(spark, "C:\\Users\MoemenEbdelli\Desktop\oumaima\bundles")
>>> observations = extract_entry(spark, bundles, 'Patient').cache()
the last command invoked an ERROR:
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "C:\bunsen\python\bunsen\r4\bundles.py", line 101, in extract_entry
bundles.extractEntry(sparkSession._jsparkSession, javaRDD, resourceName, jArray),
File "C:\spark\spark-3.2.0-bin-hadoop3.2\python\lib\py4j-0.10.9.2-src.zip\py4j\java_gateway.py", line 1309, in __call__
File "C:\spark\spark-3.2.0-bin-hadoop3.2\python\pyspark\sql\utils.py", line 111, in deco
return f(*a, **kw)
File "C:\spark\spark-3.2.0-bin-hadoop3.2\python\lib\py4j-0.10.9.2-src.zip\py4j\protocol.py", line 326, in get_return_value
py4j.protocol.Py4JJavaError: An error occurred while calling o37.extractEntry.
: java.lang.NoSuchMethodError: org.apache.spark.sql.catalyst.expressions.objects.Invoke$.apply$default$5()Z
at com.cerner.bunsen.EncoderBuilder.com$cerner$bunsen$EncoderBuilder$$childToExpr(EncoderBuilder.scala:259)
at com.cerner.bunsen.EncoderBuilder$$anonfun$6.apply(EncoderBuilder.scala:307)
at com.cerner.bunsen.EncoderBuilder$$anonfun$6.apply(EncoderBuilder.scala:307)
at scala.collection.TraversableLike.$anonfun$flatMap$1(TraversableLike.scala:293)
at scala.collection.Iterator.foreach(Iterator.scala:943)
at scala.collection.Iterator.foreach$(Iterator.scala:943)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1431)
at scala.collection.IterableLike.foreach(IterableLike.scala:74)
at scala.collection.IterableLike.foreach$(IterableLike.scala:73)
at scala.collection.AbstractIterable.foreach(Iterable.scala:56)
at scala.collection.TraversableLike.flatMap(TraversableLike.scala:293)
at scala.collection.TraversableLike.flatMap$(TraversableLike.scala:290)
at scala.collection.AbstractTraversable.flatMap(Traversable.scala:108)
at com.cerner.bunsen.EncoderBuilder.com$cerner$bunsen$EncoderBuilder$$serializer(EncoderBuilder.scala:307)
at com.cerner.bunsen.EncoderBuilder$.of(EncoderBuilder.scala:53)
at com.cerner.bunsen.EncoderBuilder.of(EncoderBuilder.scala)
at com.cerner.bunsen.FhirEncoders.of(FhirEncoders.java:213)
at com.cerner.bunsen.FhirEncoders.of(FhirEncoders.java:262)
at com.cerner.bunsen.Bundles.extractEntry(Bundles.java:291)
at com.cerner.bunsen.Bundles.extractEntry(Bundles.java:235)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
at py4j.Gateway.invoke(Gateway.java:282)
at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
at py4j.commands.CallCommand.execute(CallCommand.java:79)
at py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
at py4j.ClientServerConnection.run(ClientServerConnection.java:106)
at java.lang.Thread.run(Thread.java:748)