Pyspark's pprint not printing in jupyter

Question

I'm running a Spark streaming program on jupyter to consume tweets from a Kafka cluster and I'm facing troubles getting the following code to print the count of tweets properly.

import os
os.environ['PYSPARK_SUBMIT_ARGS'] = '--packages org.apache.spark:spark-streaming-kafka-0-8_2.11:2.3.0 pyspark-shell'

from pyspark import SparkContext
#    Spark Streaming
from pyspark.streaming import StreamingContext
#    Kafka
from pyspark.streaming.kafka import KafkaUtils
#    json parsing
import json
import logging

sc = SparkContext(appName="PythonSparkStreamingKafka_RM_01")
sc.setLogLevel("INFO")

ssc = StreamingContext(sc, 60)

kafkaStream = KafkaUtils.createStream(ssc, 'cdh57-01-node-01.moffatt.me:2181', 'spark-streaming', {'kafkaspark':1})

parsed = kafkaStream.map(lambda v: json.loads(v[1]))

parsed.count().map(lambda x:'Tweets in this batch: %s' % x).pprint()

ssc.start()
ssc.awaitTermination(timeout=180)

And this is what I get as output after running ssc.start() and ssc.awaitTermination(timeout=180)

-------------------------------------------
Time: 2020-10-22 19:48:00
-------------------------------------------

-------------------------------------------
Time: 2020-10-22 19:49:00
-------------------------------------------

-------------------------------------------
Time: 2020-10-22 19:50:00
-------------------------------------------

Only the time gets printed out and not the count of tweets . What is causing this problem ? The extracted tweets are seen in my kafka consumer which is running in the terminal.

This is my code for Kafka cluster :

import pykafka
import json
import tweepy
from tweepy import OAuthHandler
from tweepy import Stream
from tweepy.streaming import StreamListener

#TWITTER API CONFIGURATIONS
consumer_key = '--key--'
consumer_secret = '--key--'
access_token = '--key--'
access_secret = '--key--'

#TWITTER API AUTH
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_secret)

api = tweepy.API(auth)

#Twitter Stream Listener
class KafkaPushListener(StreamListener):          
    def __init__(self):
        #localhost:9092 = Default Zookeeper Producer Host and Port Adresses
        self.client = pykafka.KafkaClient("localhost:9092")

        #Get Producer that has topic name is Twitter
        self.producer = self.client.topics[bytes("kafkaspark", "ascii")].get_producer()
  
    def on_data(self, data):
        #Producer produces data for consumer
        #Data comes from Twitter
        self.producer.produce(bytes(data, "ascii"))
        return True
                                                                                                                                           
    def on_error(self, status):
        print(status)
        return True

#Twitter Stream Config
twitter_stream = Stream(auth, KafkaPushListener())

#Produce Data that has hashtag (Tweets)
twitter_stream.filter(track=['#Coldplay'])

I'm a beginner to Kafka and Spark and have no idea what is causing this . Can someone please help me with this ?

Pyspark's pprint not printing in jupyter

0 Answers