Setting timer to different timers for different elements does not fire as expected

Viewed 28

I am trying to code an example that shows how timers and state work in ApacheBeam and test this code using beam playground. I read a list of numbers, store them in KV as divisibleByTwo and divisibleByThree. Then I run a statefulDoFn with this KV. I have set the timer to expire after x seconds for numbers divisible by 2 and y seconds for numbers divisible by 3. However the test does not print them after the x and y seconds. Am I missing something in the test case?

/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements.  See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership.  The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License.  You may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.beam.examples;
 
import java.util.Arrays;
import java.util.List;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.io.TextIO;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.transforms.Count;
import org.apache.beam.sdk.transforms.Filter;
import org.apache.beam.sdk.transforms.FlatMapElements;
import org.apache.beam.sdk.transforms.MapElements;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.TypeDescriptors;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.values.TupleTag;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.transforms.Create;
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.values.PCollectionTuple;
import org.apache.beam.sdk.values.TupleTagList;
import org.apache.beam.sdk.transforms.Flatten;
import org.apache.beam.sdk.values.PCollectionList;
import org.apache.beam.sdk.state.BagState;
import org.apache.beam.sdk.state.StateSpec;
import org.apache.beam.sdk.state.StateSpecs;
import org.apache.beam.sdk.state.TimeDomain;
import org.apache.beam.sdk.state.Timer;
import org.apache.beam.sdk.state.TimerMap;
import org.apache.beam.sdk.state.TimerSpec;
import org.apache.beam.sdk.state.TimerSpecs;
import org.joda.time.Duration;
import java.time.Instant; 
 
 
public class TestClass {
 
  public static void main(String[] args) {
 
    PipelineOptions options = PipelineOptionsFactory.create();
 
    // Create the Pipeline object with the options we defined above
    Pipeline p = Pipeline.create(options);
    List<Integer> integerList = Arrays.asList(1,2,3,4,5,6,7);
    PCollection<Integer> initialCollection = p.apply(Create.of(integerList));
    PCollection<KV<String, Integer>> outputPCollection = initialCollection.apply(ParDo.of(new SplitDoFn()))
      .apply("Printing Elements", ParDo.of(new DoFn<KV<String, Integer>, KV<String, Integer>>() {
          @ProcessElement
          public void processElement(ProcessContext c) {
            System.out.print(c.element().getKey() + " is " + c.element().getValue() +" \n");
            c.output(c.element());
          }
        }));
   
    outputPCollection.apply(ParDo.of(new StateDoFn()));
    p.run().waitUntilFinish();
  }
  
  static class StateDoFn extends DoFn<KV<String, Integer>, Integer> {
    @StateId("buffer")
    private final StateSpec<BagState<Integer>> bufferedEvents = StateSpecs.bag();
    
    @TimerId("expiry")
    private final TimerSpec expirySpec = TimerSpecs.timer(TimeDomain.PROCESSING_TIME);
    
    @ProcessElement
    public void process(ProcessContext c, 
    @StateId("buffer") BagState<Integer> bufferState,
    @TimerId("expiry") Timer expiryTimer) {
      String key = c.element().getKey();
      
      if(key.equals("divisibleByTwo")) {
        expiryTimer.offset(Duration.standardSeconds(60)).setRelative();
        System.out.println("Scheduling element " + c.element().getValue() + " to be printed at " + Instant.now() + Duration.standardSeconds(60)); 
        bufferState.add(c.element().getValue());
      } else if(key.equals("divisibleByThree")) {
        expiryTimer.offset(Duration.standardSeconds(120)).setRelative();
        System.out.println("Scheduling element " + c.element().getValue() + " to be printed at " + Instant.now() + Duration.standardSeconds(120)); 
        bufferState.add(c.element().getValue());
      }
      
    }
    
    @OnTimer("expiry")
    public void onExpiry(
    OnTimerContext context,
    @StateId("buffer") BagState<Integer> bufferState) {
    if(bufferState.isEmpty().read()) {
       System.out.println("ERROR"); 
    }
      
    if (!bufferState.isEmpty().read()) {
      for (Integer value : bufferState.read()) {
        System.out.println(value);
        context.output(value);
      }
      bufferState.clear();
      }
    }
  }
 
  static class SplitDoFn extends DoFn<Integer, KV<String, Integer>> {
    @ProcessElement
    public void process(ProcessContext context) {
      Integer element = context.element();
      
      if(element % 2 == 0) {
         context.output(KV.of("divisibleByTwo", element));
      }
      
      if(element % 3 == 0) {
         context.output(KV.of("divisibleByThree", element));
      }
    }
    
  }
} ```
1 Answers

There is just one timer, so when you set it again it replaces the prior value.

Related