I am trying to code an example that shows how timers and state work in ApacheBeam and test this code using beam playground. I read a list of numbers, store them in KV as divisibleByTwo and divisibleByThree. Then I run a statefulDoFn with this KV. I have set the timer to expire after x seconds for numbers divisible by 2 and y seconds for numbers divisible by 3. However the test does not print them after the x and y seconds. Am I missing something in the test case?
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.beam.examples;
import java.util.Arrays;
import java.util.List;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.io.TextIO;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.transforms.Count;
import org.apache.beam.sdk.transforms.Filter;
import org.apache.beam.sdk.transforms.FlatMapElements;
import org.apache.beam.sdk.transforms.MapElements;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.TypeDescriptors;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.values.TupleTag;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.transforms.Create;
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.values.PCollectionTuple;
import org.apache.beam.sdk.values.TupleTagList;
import org.apache.beam.sdk.transforms.Flatten;
import org.apache.beam.sdk.values.PCollectionList;
import org.apache.beam.sdk.state.BagState;
import org.apache.beam.sdk.state.StateSpec;
import org.apache.beam.sdk.state.StateSpecs;
import org.apache.beam.sdk.state.TimeDomain;
import org.apache.beam.sdk.state.Timer;
import org.apache.beam.sdk.state.TimerMap;
import org.apache.beam.sdk.state.TimerSpec;
import org.apache.beam.sdk.state.TimerSpecs;
import org.joda.time.Duration;
import java.time.Instant;
public class TestClass {
public static void main(String[] args) {
PipelineOptions options = PipelineOptionsFactory.create();
// Create the Pipeline object with the options we defined above
Pipeline p = Pipeline.create(options);
List<Integer> integerList = Arrays.asList(1,2,3,4,5,6,7);
PCollection<Integer> initialCollection = p.apply(Create.of(integerList));
PCollection<KV<String, Integer>> outputPCollection = initialCollection.apply(ParDo.of(new SplitDoFn()))
.apply("Printing Elements", ParDo.of(new DoFn<KV<String, Integer>, KV<String, Integer>>() {
@ProcessElement
public void processElement(ProcessContext c) {
System.out.print(c.element().getKey() + " is " + c.element().getValue() +" \n");
c.output(c.element());
}
}));
outputPCollection.apply(ParDo.of(new StateDoFn()));
p.run().waitUntilFinish();
}
static class StateDoFn extends DoFn<KV<String, Integer>, Integer> {
@StateId("buffer")
private final StateSpec<BagState<Integer>> bufferedEvents = StateSpecs.bag();
@TimerId("expiry")
private final TimerSpec expirySpec = TimerSpecs.timer(TimeDomain.PROCESSING_TIME);
@ProcessElement
public void process(ProcessContext c,
@StateId("buffer") BagState<Integer> bufferState,
@TimerId("expiry") Timer expiryTimer) {
String key = c.element().getKey();
if(key.equals("divisibleByTwo")) {
expiryTimer.offset(Duration.standardSeconds(60)).setRelative();
System.out.println("Scheduling element " + c.element().getValue() + " to be printed at " + Instant.now() + Duration.standardSeconds(60));
bufferState.add(c.element().getValue());
} else if(key.equals("divisibleByThree")) {
expiryTimer.offset(Duration.standardSeconds(120)).setRelative();
System.out.println("Scheduling element " + c.element().getValue() + " to be printed at " + Instant.now() + Duration.standardSeconds(120));
bufferState.add(c.element().getValue());
}
}
@OnTimer("expiry")
public void onExpiry(
OnTimerContext context,
@StateId("buffer") BagState<Integer> bufferState) {
if(bufferState.isEmpty().read()) {
System.out.println("ERROR");
}
if (!bufferState.isEmpty().read()) {
for (Integer value : bufferState.read()) {
System.out.println(value);
context.output(value);
}
bufferState.clear();
}
}
}
static class SplitDoFn extends DoFn<Integer, KV<String, Integer>> {
@ProcessElement
public void process(ProcessContext context) {
Integer element = context.element();
if(element % 2 == 0) {
context.output(KV.of("divisibleByTwo", element));
}
if(element % 3 == 0) {
context.output(KV.of("divisibleByThree", element));
}
}
}
} ```