I'm using spark base image:spark:v3.0.2 in Dockerfile. The docker is supposed to run a jar.
and in the jar using I tried using spark-version=3.0.0 , 3.1.0 (scala-version=2.12) with both getting the below exception when starting the jar
listOf("spark-core", "spark-sql", "spark-kubernetes", "spark-streaming-kafka-0-10-assembly").forEach { name ->
implementation("org.apache.spark:${name}_$scalaVersion:$sparkVersion")
}
What am I missing, any pointers would help
{"time":"","level":"ERROR","thread":"main","logger":"org.apache.spark.SparkContext","context":"default","message":"Error initializing SparkContext.","stack_trace":"j.u.ServiceConfigurationError: org.apache.spark.scheduler.ExternalClusterManager: **org.apache.spark.scheduler.cluster.mesos.MesosClusterManager not a subtype\n\tat** j.util.ServiceLoader.fail(Unknown Source)\n\tat j.u.ServiceLoader$LazyClassPathLookupIterator.hasNextService(Unknown Source)\n\tat j.u.ServiceLoader$LazyClassPathLookupIterator.hasNext(Unknown Source)\n\tat j.u.ServiceLoader$2.hasNext(Unknown Source)\n\tat j.u.ServiceLoader$3.hasNext(Unknown Source)\n\tat s.c.c.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:43)\n\tat s.c.Iterator.foreach(Iterator.scala:943)\n\tat s.c.Iterator.foreach$(Iterator.scala:943)\n\tat s.c.AbstractIterator.foreach(Iterator.scala:1431)\n\tat s.c.IterableLike.foreach(IterableLike.scala:74)\n\tat s.c.IterableLike.foreach$(IterableLike.scala:73)\n\tat s.c.AbstractIterable.foreach(Iterable.scala:56)\n\tat s.c.TraversableLike.filterImpl(TraversableLike.scala:302)\n\tat s.c.TraversableLike.filterImpl$(TraversableLike.scala:296)\n\tat s.c.AbstractTraversable.filterImpl(Traversable.scala:108)\n\tat s.c.TraversableLike.filter(TraversableLike.scala:394)\n\tat s.c.TraversableLike.filter$(TraversableLike.scala:394)\n\tat s.c.AbstractTraversable.filter(Traversable.scala:108)\n\tat o.a.s.SparkContext$.getClusterManager(SparkContext.scala:2942)\n\tat o.a.s.SparkContext$.org$apache$spark$SparkContext$$createTaskScheduler(SparkContext.scala:2922)\n\tat o.a.s.SparkContext.<init>(SparkContext.scala:528)\n\tat o.a.s.SparkContext$.getOrCreate(SparkContext.scala:2555)\n\tat o.a.s.s.SparkSession$Builder.$anonfun$getOrCreate$1(SparkSession.scala:930)\n\tat scala.Option.getOrElse(Option.scala:189)\n\tat o.a.s.s.SparkSession$Builder.getOrCreate(SparkSession.scala:921)\n\tat c.s.n.j.c.SparkProvider.createSparkSession(SparkProvider.scala:27)\n\tat c.s.n.j.c.SparkProvider.<init>(SparkProvider.scala:10)\n\tat c.s.n.j.t.SparkTask.<init>(SparkTask.scala:7)\n\tat c.s.n.j.s.StatementNotificationTask.<init>(StatementNotificationTask.scala:13)\n\tat j.i.r.NativeConstructorAccessorImpl.newInstance0(Unknown Source)\n\t... 19 frames truncated\n"}
{"time":"2021-02-19T10:48:43.536+0000","level":"WARN","thread":"main","logger":"org.apache.spark.metrics.MetricsSystem","context":"default","message":"Stopping a MetricsSystem that is not running"}
Exception in thread "main" java.util.ServiceConfigurationError: org.apache.spark.scheduler.ExternalClusterManager: org.apache.spark.scheduler.cluster.mesos.MesosClusterManager not a subtype
at java.base/java.util.ServiceLoader.fail(Unknown Source)
at java.base/java.util.ServiceLoader$LazyClassPathLookupIterator.hasNextService(Unknown Source)
at java.base/java.util.ServiceLoader$LazyClassPathLookupIterator.hasNext(Unknown Source)
at java.base/java.util.ServiceLoader$2.hasNext(Unknown Source)
at java.base/java.util.ServiceLoader$3.hasNext(Unknown Source)
at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:43)
at scala.collection.Iterator.foreach(Iterator.scala:943)
at scala.collection.Iterator.foreach$(Iterator.scala:943)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1431)
at scala.collection.IterableLike.foreach(IterableLike.scala:74)
at scala.collection.IterableLike.foreach$(IterableLike.scala:73)
at scala.collection.AbstractIterable.foreach(Iterable.scala:56)
at scala.collection.TraversableLike.filterImpl(TraversableLike.scala:302)
at scala.collection.TraversableLike.filterImpl$(TraversableLike.scala:296)
at scala.collection.AbstractTraversable.filterImpl(Traversable.scala:108)
at scala.collection.TraversableLike.filter(TraversableLike.scala:394)