TypeError: 'JavaPackage' object is not callable | using java 11 for spark 3.3.0, sparknlp 4.0.1 and sparknlp jar from spark-nlp-m1_2.12

Question

spark nlp jar, I got it from https://jar-download.com/artifacts/com.johnsnowlabs.nlp/spark-nlp-m1_2.12/4.0.1/source-code

JAVA_HOME = C:\Program Files\Java\jdk-18.0.1.1 In the system variables and users admin variables.

'''

import pyspark
from pyspark import SparkContext
from pyspark.sql import SparkSession
import sparknlp
from sparknlp.base import *
from sparknlp.annotator import *
from pyspark.ml import Pipeline
import pandas as pd
import warnings
warnings.filterwarnings('ignore')


sc = SparkContext(master='local[2]')

spark = SparkSession.builder.appName('test')\
    .config("spark.jars", "/Users/Admin/Anaconda3/Lib/site-packages/sparknlp/lib/jar_files.jar")\
    .config("spark.driver.extraClassPath", "/Users/Admin/Anaconda3/Lib/site-packages/sparknlp/lib/jar_files.jar")\
    .config("spark.executor.extraClassPath", "/Users/Admin/Anaconda3/Lib/site-packages/sparknlp/lib/jar_files.jar")\
    .getOrCreate()

print("Spark NLP version", sparknlp.version())
print("Apache Spark version:", spark.version)

document = DocumentAssembler().setInputCol("description").setOutputCol("document")


**Executed the above**

**Throws the following error**

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
~\AppData\Local\Temp/ipykernel_6864/2064948474.py in 
----> 1 **document = DocumentAssembler().setInputCol("description").setOutputCol("document")**

~\Anaconda3\lib\site-packages\pyspark\__init__.py in wrapper(self, *args, **kwargs)
    133             raise TypeError("Method %s forces keyword arguments." % func.__name__)
    134         self._input_kwargs = kwargs
--> 135         return func(self, **kwargs)
    136 
    137     return cast(F, wrapper)

~\Anaconda3\lib\site-packages\sparknlp\base\document_assembler.py in __init__(self)
     90     @keyword_only
     91     def __init__(self):
---> 92         super(DocumentAssembler, self).__init__(classname="com.johnsnowlabs.nlp.DocumentAssembler")
     93         self._setDefault(outputCol="document", cleanupMode='disabled')
     94 

~\Anaconda3\lib\site-packages\pyspark\__init__.py in wrapper(self, *args, **kwargs)
    133             raise TypeError("Method %s forces keyword arguments." % func.__name__)
    134         self._input_kwargs = kwargs
--> 135         return func(self, **kwargs)
    136 
    137     return cast(F, wrapper)

~\Anaconda3\lib\site-packages\sparknlp\internal\annotator_transformer.py in __init__(self, classname)
     31         self.setParams(**kwargs)
     32         self.__class__._java_class_name = classname
---> 33         self._java_obj = self._new_java_obj(classname, self.uid)
     34 

~\Anaconda3\lib\site-packages\pyspark\ml\wrapper.py in _new_java_obj(java_class, *args)
     84             java_obj = getattr(java_obj, name)
     85         java_args = [_py2java(sc, arg) for arg in args]
---> 86         return java_obj(*java_args)
     87 
     88     @staticmethod

**TypeError: 'JavaPackage' object is not callable**

'''

These are the contents inside the jar file https://i.stack.imgur.com/GlIgD.png

user_3357812 · Answer

Instead of downloading and then referencing the jar, you can give the spark session your maven coordinates. This works for me:

SPARK_JARS = ["com.google.cloud.spark:spark-bigquery-with-dependencies_2.12:0.26.0",
              "com.johnsnowlabs.nlp:spark-nlp_2.12:4.1.0"]

def get_spark(master="local[*]", name="Colab"):
    builder = SparkSession.builder.appName(name)
    builder.config('spark.ui.port', '4050')
    builder.config('spark.jars.packages', ",".join(SPARK_JARS))
    builder.config("spark.driver.memory", "16G")
    builder.config("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
    builder.config("spark.kryoserializer.buffer.max", "2000M")
    builder.config("spark.driver.maxResultSize", "0")
    return builder.getOrCreate()

Then, create your spark session with:

spark = get_spark()

BTW, the following has the above-referenced jars: https://mvnrepository.com/artifact/com.johnsnowlabs.nlp/spark-nlp_2.12/4.1.0

TypeError: 'JavaPackage' object is not callable | using java 11 for spark 3.3.0, sparknlp 4.0.1 and sparknlp jar from spark-nlp-m1_2.12

1 Answers