Unable to recognize avg and over function in IntelliJ. It is saying cannot resolve symbol avg and over. Can someone please say which library to import.
object DomainSpecificLanguage {
def main(args: Array[String]): Unit = {
System.setProperty("hadoop.home.dir", "C:/winutils")
val spark = SparkSession.builder().appName("DomainSpecificLanguage").config("spark.master" , "local").getOrCreate()
spark.sparkContext.setLogLevel("ERROR")
import spark.implicits._
val empSalary =Seq(
Salary("sales", 1, 5000),
Salary("personnel", 2, 3900),
Salary("sales", 3, 4800),
Salary("sales", 4, 4800),
Salary("personnel", 5, 3500),
Salary("develop", 7, 4200),
Salary("develop", 8, 6000),
Salary("develop", 9, 4500),
Salary("develop", 10, 5200),
Salary("develop", 11, 5200)).toDS
val byDepName = Window.partitionBy('depName)
empSalary.withColumn("avg", avg('salary) over byDepName)
}
}