I am trying to read data from websocket using Apache Flink
My Flink job is connecting to websocket but it is not pulling data from websocket.
below is the sample code that I have tried to connect to websocket using Apache flink API
the run() method in the RichSourceFunction neither executing nor throwing any error.
@Slf4j
public class Main {
public static final int CHECKPOINTING_INTERVAL_MS = 5000;
private static final String JOB_NAME = "Flink Streaming Java API Skeleton";
/**
* Main Flink job.
*
* @param args
* @throws Exception
*/
public static void main(String[] args) throws Exception {
// set up the streaming execution environment
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
final ObjectMapper objectMapper = new ObjectMapper();
env.setParallelism(4);
ParameterTool paramTool = ParameterTool.fromArgs(args);
env.getConfig().setGlobalJobParameters(paramTool);
DataStreamSource<String> mySocketStream = env.addSource(new MyWebSocketSourceFunc());
mySocketStream.map(new MapIt()).print();
// mySocketStream.print();
env.enableCheckpointing(CHECKPOINTING_INTERVAL_MS);
env.setRestartStrategy(RestartStrategies.noRestart());
env.execute(JOB_NAME);
}
public static class MyWebSocketSourceFunc extends RichSourceFunction<String> {
private boolean running = true;
transient AsyncHttpClient client;
transient BoundRequestBuilder boundRequestBuilder;
transient WebSocketUpgradeHandler.Builder webSocketListener;
private BlockingQueue<String> messages = new ArrayBlockingQueue<>(100);
@Override
public void run(SourceContext<String> ctx) throws Exception {
WebSocketUpgradeHandler webSocketUpgradeHandler = webSocketListener.addWebSocketListener(
new WebSocketListener() {
private final ObjectMapper myMapper = new ObjectMapper();
private String getRsvpId(String payload) {
try {
Map map = myMapper.readValue(payload, Map.class);
Object rsvpId = map.get("rsvp_id");
return rsvpId != null ? rsvpId.toString() : "NOT FOUND";
} catch (IOException e) {
log.error("Mapping failed, returning 'null'");
return "NULL";
}
}
@Override
public void onOpen(WebSocket webSocket) {
}
@Override
public void onClose(WebSocket webSocket, int i, String s) {
}
@Override
public void onError(Throwable throwable) {
}
@Override
public void onTextFrame(String payload, boolean finalFragment, int rsv) {
log.debug("onTextFrame({}), rsvp_id={}", hash(payload), getRsvpId(payload));
if (payload != null) {
try {
messages.put(payload);
} catch (InterruptedException e) {
log.error("Interrupted!", e);
Thread.currentThread().interrupt();
}
}
}
}).build();
boundRequestBuilder.execute(webSocketUpgradeHandler).get();
while (running) {
ctx.collect(messages.take());
}
running = false;
}
@Override
public void cancel() {
log.info("cancel function called");
running = false;
}
@Override
public void open(Configuration parameters) throws Exception {
log.info("open function called");
super.open(parameters);
client = Dsl.asyncHttpClient();
boundRequestBuilder = client.prepareGet("ws://stream.meetup.com/2/rsvps");
webSocketListener = new WebSocketUpgradeHandler.Builder();
}
private String hash(String input) {
if (input == null) {
return "-- NULL --";
}
try {
MessageDigest md = MessageDigest.getInstance("MD5");
md.update(input.getBytes());
byte[] digest = md.digest();
return DatatypeConverter.printHexBinary(digest).toUpperCase();
} catch (NoSuchAlgorithmException e) {
log.error("Cound not instantiate MD5", e);
return "--NOT CALCULATED--";
}
}
}
public static class MapIt extends RichMapFunction<String, String> {
final ObjectMapper objectMapper = new ObjectMapper();
@Override
public String map(String value) throws Exception {
Map<String, Object> mapped = objectMapper.readValue(value, Map.class);
Object rsvp = mapped.get("rsvp_id");
return rsvp != null ? rsvp.toString() : "null" ;
}
}
}
Here is the reference document that I have followed to implement this job Referance
TYIA.