Spark DynamoDB Connectivity Issue

Requirement: Read data from DynamoDB(not local but on AWS) via Spark using Scala from my local machine.

Understanding: Data can be read using the emr-hadoop-dynamodb.jar when we are using an EMR cluster

Question:

1. Can we read data from DynamoDB(on cloud and not local) using the emr-dynamodb-hadoop.jar?
2. EMR cluster is not to be used. I directly want to access dynamodb from spark using scala code on my local machine

build.sbt

version := "0.1"

scalaVersion := "2.11.12"
scalacOptions := Seq("-target:jvm-1.8")

libraryDependencies ++= Seq(
  "software.amazon.awssdk" % "dynamodb" % "2.15.1",
  "org.apache.spark" %% "spark-core" % "2.4.1",
  "com.amazon.emr" % "emr-dynamodb-hadoop" % "4.2.0",
  "org.apache.httpcomponents" % "httpclient" % "4.5"
)

dependencyOverrides ++= {
  Seq(
    "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.6.7.1",
    "com.fasterxml.jackson.core" % "jackson-databind" % "2.6.7",
    "com.fasterxml.jackson.core" % "jackson-core" % "2.6.7"
  )
}

readDataFromDDB.scala

import org.apache.hadoop.dynamodb.DynamoDBItemWritable
import org.apache.hadoop.dynamodb.read.DynamoDBInputFormat
import org.apache.hadoop.io.Text
import org.apache.hadoop.mapred.JobConf
import org.apache.spark.api.java.JavaSparkContext
import org.apache.spark.{SparkConf, SparkContext}

object readDataFromDDB {
  def main(args: Array[String]): Unit = {
    var sc: SparkContext = null
    try {
      val conf = new SparkConf().setAppName("DynamoDBApplication").setMaster("local")
      sc = new SparkContext(conf)
      val jobConf = getDynamoDbJobConf(sc, "Music", "TableNameForWrite")
      val tableData = sc.hadoopRDD(jobConf, classOf[DynamoDBInputFormat], classOf[Text], classOf[DynamoDBItemWritable])
      println(tableData.count())

    } catch {
      case e: Exception => {
        println(e.getStackTrace)
      }
    } finally {
      sc.stop()
    }
  }

  private def getDynamoDbJobConf(sc: JavaSparkContext, tableNameForRead: String, tableNameForWrite: String) = {
    val jobConf = new JobConf(sc.hadoopConfiguration)
    jobConf.set("dynamodb.servicename", "dynamodb")
    jobConf.set("dynamodb.input.tableName", tableNameForRead)
    jobConf.set("dynamodb.output.tableName", tableNameForWrite)
    jobConf.set("dynamodb.awsAccessKeyId", "*****************")
    jobConf.set("dynamodb.awsSecretAccessKey", "*********************")
    jobConf.set("dynamodb.endpoint", "dynamodb.us-east-1.amazonaws.com")
    jobConf.set("dynamodb.regionid", "us-east-1")
    jobConf.set("mapred.output.format.class", "org.apache.hadoop.dynamodb.write.DynamoDBOutputFormat")
    jobConf.set("mapred.input.format.class", "org.apache.hadoop.dynamodb.read.DynamoDBInputFormat")
    jobConf
  }
}

ERROR

java.lang.RuntimeException: Could not lookup table Music in DynamoDB. at org.apache.hadoop.dynamodb.DynamoDBClient.describeTable(DynamoDBClient.java:116) at org.apache.hadoop.dynamodb.read.ReadIopsCalculator.getThroughput(ReadIopsCalculator.java:67) at org.apache.hadoop.dynamodb.read.ReadIopsCalculator.calculateTargetIops(ReadIopsCalculator.java:57) at org.apache.hadoop.dynamodb.read.AbstractDynamoDBRecordReader.initReadManager(AbstractDynamoDBRecordReader.java:153) at org.apache.hadoop.dynamodb.read.AbstractDynamoDBRecordReader.(AbstractDynamoDBRecordReader.java:84) at org.apache.hadoop.dynamodb.read.DefaultDynamoDBRecordReader.(DefaultDynamoDBRecordReader.java:24) at org.apache.hadoop.dynamodb.read.DynamoDBInputFormat.getRecordReader(DynamoDBInputFormat.java:32) at org.apache.spark.rdd.HadoopRDD$$anon$1.liftedTree1$1(HadoopRDD.scala:267) at org.apache.spark.rdd.HadoopRDD$$anon$1.(HadoopRDD.scala:266) at org.apache.spark.rdd.HadoopRDD.compute(HadoopRDD.scala:224) at org.apache.spark.rdd.HadoopRDD.compute(HadoopRDD.scala:95) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:324) at org.apache.spark.rdd.RDD.iterator(RDD.scala:288) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90) at org.apache.spark.scheduler.Task.run(Task.scala:121) at org.apache.spark.executor.Executor$TaskRunner$$anonfun$10.apply(Executor.scala:403) at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1360) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:409) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: Socket not created by this factory at org.apache.hadoop.dynamodb.DynamoDBFibonacciRetryer.handleException(DynamoDBFibonacciRetryer.java:120) at org.apache.hadoop.dynamodb.DynamoDBFibonacciRetryer.runWithRetry(DynamoDBFibonacciRetryer.java:83) at org.apache.hadoop.dynamodb.DynamoDBClient.describeTable(DynamoDBClient.java:105) ... 20 more

Links already reviewed

https://aws.amazon.com/blogs/big-data/analyze-your-data-on-amazon-dynamodb-with-apache-spark/

read/write dynamo db from apache spark

Spark 2.2.0 - How to write/read DataFrame to DynamoDB

https://github.com/awslabs/emr-dynamodb-connector