Below are 2 queries for the code
import org.apache.spark.sql._
val columns = Seq("language","users_count")
val data = Seq(("Java", "20000"), ("Python", "100000"), ("Scala", "3000"))
val rdd = spark.sparkContext.parallelize(data)
val dfFromRDD1 = rdd.toDF("language","users_count")
val data1 = Seq(("Java"), ("Python"), ("Scala"))
val rdd1 = spark.sparkContext.parallelize(data1)
val dfFromRDD2 = rdd1.toDF("language")
cached:-
dfFromRDD1.join(dfFromRDD2,dfFromRDD1.col("language").lt(dfFromRDD2.col("language")),"inner").cache().show()
Without Caching:-
dfFromRDD1.join(dfFromRDD2,dfFromRDD1.col("language").lt(dfFromRDD2.col("language")),"inner").show()
We get a beautiful visual query plan for non cached computation but not for cached one. Want to understand, how can we see the visual query plan for cached computation?
