Hello I am triyin to apply two aggregate in spark datastream, with the following code
from pyspark.sql.functions import udf
from pyspark.sql.types import StringType, FloatType
@udf(returnType=FloatType())
def sent_points(x):
words = x.split(" ")
total = sum([negative.get(word, 0) + positive.get(word, 0) for word in words])
return total
@udf(returnType=StringType())
def sent_type(x):
x = sent_points(x)
if x == 0:
return 'NEUTRO'
if x > 0:
return 'POSITIVE'
if x < 0:
return 'NEGATIVE'
ds = df.selectExpr("CAST(value AS STRING)", "timestamp") \
.withWatermark("timestamp", "10 seconds") \
.groupBy('timestamp', 'value') \
.agg(sent_points(col('value')).alias('points')) \
.selectExpr('value', 'points', 'timestamp') \
.agg(sent_type(col('points')).alias('tipe'))
But I am going the following error:
TypeError: cannot pickle '_thread.RLock' object
This work fine if I remove the second aggregation but I need that value too.
How can I solve this, in order to get points and tipe values.
Thanks