Java Code Examples for com.datatorrent.api.annotation.Stateless#WINDOW_ID

@Override
public void beginWindow(long windowId)
{
  currentWindowTupleCount = 0;
  currentWindowId = windowId;
  if ( currentWindowId != Stateless.WINDOW_ID) {
    if (currentWindowId > reconcilingPhaseWindowId) {
      isCurrentlyInSafeMode = false;
      isCurrentlyInReconcilingMode = false;
    }
    if (currentWindowId == reconcilingPhaseWindowId) {
      isCurrentlyInReconcilingMode = true;
      isCurrentlyInSafeMode = false;
    }
    if (currentWindowId < reconcilingPhaseWindowId) {
      isCurrentlyInReconcilingMode = false;
      isCurrentlyInSafeMode = true;
    }
  }
  LOG.info(" Current processing mode states Safe Mode = " + isCurrentlyInSafeMode + " Reconciling mode = "
      + isCurrentlyInReconcilingMode);
  LOG.info(" Current window ID = " + currentWindowId + " reconciling window ID = " + reconcilingPhaseWindowId);
}
 

/**
 * This method checkpoints the given writer.
 * @param writer The writer to checkpoint.
 * @return new writer.
 */
public static AbstractFileOutputOperator checkpoint(AbstractFileOutputOperator writer, long windowId)
{
  if (windowId >= Stateless.WINDOW_ID) {
    writer.beforeCheckpoint(windowId);
  }
  Kryo kryo = new Kryo();
  ByteArrayOutputStream bos = new ByteArrayOutputStream();
  Output loutput = new Output(bos);
  kryo.writeObject(loutput, writer);
  loutput.close();

  Input lInput = new Input(bos.toByteArray());
  @SuppressWarnings("unchecked")
  AbstractFileOutputOperator checkPointedWriter = kryo.readObject(lInput, writer.getClass());
  lInput.close();

  return checkPointedWriter;
}
 

@Override
public void setup(Context.OperatorContext context)
{
  currentWindowRecoveryState = Lists.newLinkedList();
  if (referenceTimes == null) {
    referenceTimes = new ConcurrentHashMap<>();
  }
  scanner.setup(context);
  windowDataManager.setup(context);
  super.setup(context);

  long largestRecoveryWindow = windowDataManager.getLargestCompletedWindow();
  if (largestRecoveryWindow == Stateless.WINDOW_ID || context.getValue(Context.OperatorContext.ACTIVATION_WINDOW_ID) >
      largestRecoveryWindow) {
    scanner.startScanning(Collections.unmodifiableMap(referenceTimes));
  }
}
 

@Override
public void beginWindow(long windowId)
{
  super.beginWindow(windowId);
  currentWindowId = windowId;
  if ( currentWindowId != Stateless.WINDOW_ID) { // if it is not the first window of the application
    if (currentWindowId > reconcilingWindowId) {
      isInReplayMode = false;
      isInReconcilingMode = false;
    }
    if (currentWindowId == reconcilingWindowId) {
      isInReconcilingMode = true;
      isInReplayMode = false;
    }
    if (currentWindowId < reconcilingWindowId) {
      isInReconcilingMode = false;
      isInReplayMode = true;
    }
  }
  numDeletes = 0;
  numInserts = 0;
  numUpdates = 0;
  numUpserts = 0;
}
 

private void initCheckpoint(PTOperator oper, Operator oo, Checkpoint checkpoint)
{
  try {
    LOG.debug("Writing activation checkpoint {} {} {}", checkpoint, oper, oo);
    long windowId = oper.isOperatorStateLess() ? Stateless.WINDOW_ID : checkpoint.windowId;
    StorageAgent agent = oper.operatorMeta.getValue(OperatorContext.STORAGE_AGENT);
    agent.save(oo, oper.id, windowId);
    if (agent instanceof AsyncStorageAgent) {
      ((AsyncStorageAgent)agent).flush(oper.id, windowId);
    }
  } catch (IOException e) {
    // inconsistent state, no recovery option, requires shutdown
    throw new IllegalStateException("Failed to write operator state after partition change " + oper, e);
  }
  oper.setRecoveryCheckpoint(checkpoint);
  if (!Checkpoint.INITIAL_CHECKPOINT.equals(checkpoint)) {
    oper.checkpoints.add(checkpoint);
  }
}
 

/**
 * Primarily resets the per window counter metrics.
 * @param windowId The windowid as provided by the apex framework
 */
@Override
public void beginWindow(long windowId)
{
  super.beginWindow(windowId);
  totalIgnoresInThisWindow = 0;
  totalWriteTimeoutsInThisWindow = 0;
  totalWriteRetriesInThisWindow =  0;
  uniqueHostsWrittenToInCurrentWindow.clear();
  successfullWrites = 0;
  ignoredRequestsDuetoIfExistsCheck = 0;
  writesWithConsistencyOne = 0;
  writesWithConsistencyTwo = 0;
  writesWithConsistencyThree = 0;
  writesWithConsistencyAll = 0;
  writesWithConsistencyLocalOne = 0;
  writesWithConsistencyQuorum = 0;
  writesWithConsistencyLocalQuorum = 0;
  writeWithConsistencyLocalSerial = 0;
  writesWithConsistencyEachQuorum = 0;
  writesWithConsistencySerial = 0;
  writesWithConsistencyAny = 0;
  currentWindowId = windowId;
  if ( currentWindowId != Stateless.WINDOW_ID) {
    if (currentWindowId > reconcilingWindowId) {
      isInSafeMode = false;
      isInReconcilingMode = false;
    }
    if (currentWindowId == reconcilingWindowId) {
      isInReconcilingMode = true;
      isInSafeMode = false;
    }
    if (currentWindowId < reconcilingWindowId) {
      isInReconcilingMode = false;
      isInSafeMode = true;
    }
  }
}
 

@Override
public void activate(OperatorContext ctx)
{
  if (context.getValue(OperatorContext.ACTIVATION_WINDOW_ID) != Stateless.WINDOW_ID &&
      context.getValue(OperatorContext.ACTIVATION_WINDOW_ID) < windowDataManager.getLargestCompletedWindow()) {
    // If it is a replay state, don't start the consumer
    return;
  }
  // Don't start thread here!
  // # of kafka_consumer_threads depends on the type of kafka client and the message
  // metadata(topic/partition/replica) layout
  consumer.start();
}
 

@Override
public void activate(Context.OperatorContext context)
{
  ApexKuduConnection.ApexKuduConnectionBuilder apexKuduConnectionBuilder = getKuduConnectionConfig();
  apexKuduConnection = apexKuduConnectionBuilder.build();
  checkNotNull(apexKuduConnection,"Kudu connection cannot be null");
  kuduTable = apexKuduConnection.getKuduTable();
  kuduSession = apexKuduConnection.getKuduSession();
  kuduClientHandle = apexKuduConnection.getKuduClient();
  checkNotNull(kuduTable,"Kudu Table cannot be null");
  checkNotNull(kuduSession, "Kudu session cannot be null");
  allColumnDefs = new HashMap();
  primaryKeyColumnNames = new HashSet<>();
  kuduColumnBasedGetters = new HashMap();
  buildGettersForPojoPayload();
  reconcilingWindowId = Stateless.WINDOW_ID;
  // The operator is working in a replay mode where the upstream buffer is re-streaming the tuples
  // Note that there are two windows that need special core. The window that is being replayed and the subsequent
  // window that might have resulted in a crash which we are referring as reconciling window
  if ( (context.getValue(Context.OperatorContext.ACTIVATION_WINDOW_ID) != Stateless.WINDOW_ID) &&
      context.getValue(Context.OperatorContext.ACTIVATION_WINDOW_ID) <
      windowDataManager.getLargestCompletedWindow()) {
    reconcilingWindowId = windowDataManager.getLargestCompletedWindow() + 1;
  }

  if ( (context.getValue(Context.OperatorContext.ACTIVATION_WINDOW_ID) != Stateless.WINDOW_ID) &&
      context.getValue(Context.OperatorContext.ACTIVATION_WINDOW_ID) ==
      windowDataManager.getLargestCompletedWindow()) {
    reconcilingWindowId = windowDataManager.getLargestCompletedWindow();
  }
}
 

/**
 * Implement ActivationListener Interface.
 */
@Override
public void activate(OperatorContext ctx)
{
  // If it is a replay state, don't start the consumer
  if (context.getValue(OperatorContext.ACTIVATION_WINDOW_ID) != Stateless.WINDOW_ID &&
      context.getValue(OperatorContext.ACTIVATION_WINDOW_ID) < windowDataManager.getLargestCompletedWindow()) {
    return;
  }
  consumer.start();
}
 

/**
 *
 * @param id the value of id
 * @param name name of the operator
 * @param attributes the value of attributes
 * @param parentContext
 */
public OperatorContext(int id, @NotNull String name, AttributeMap attributes, Context parentContext)
{
  super(attributes, parentContext);
  this.lastProcessedWindowId = Stateless.WINDOW_ID;
  this.id = id;
  this.name = Preconditions.checkNotNull(name, "operator name");
  this.stateless = super.getValue(OperatorContext.STATELESS);
}
 

@Override
public void setup(Context.OperatorContext context)
{
  if (windowDataManager != null) {
    windowDataManager.setup(context);
  }
  try {
    buildColumnSchemaForTable();
  } catch (Exception e) {
    throw new RuntimeException("Error while trying to build the schema definition for the Kudu table " + tableName,
        e);
  }
  windowManagerDataForScans = new HashMap<>();
  optionsEnabledForCurrentQuery = new HashMap<>();
  initPartitioner();
  initBuffer();
  // Scanner can only be initialized after initializing the partitioner
  initScanner(); // note that this is not streaming any data yet. Only warming up the scanner ready to read data
  initCurrentState();
  isCurrentlyInSafeMode = false;
  isCurrentlyInReconcilingMode = false;
  reconcilingPhaseWindowId = Stateless.WINDOW_ID;
  if ( (context.getValue(Context.OperatorContext.ACTIVATION_WINDOW_ID) != Stateless.WINDOW_ID) &&
      context.getValue(Context.OperatorContext.ACTIVATION_WINDOW_ID) <
      windowDataManager.getLargestCompletedWindow()) {
    isCurrentlyInSafeMode = true;
    reconcilingPhaseWindowId = windowDataManager.getLargestCompletedWindow() + 1;
    LOG.info("Set reconciling window ID as " + reconcilingPhaseWindowId);
    isCurrentlyInReconcilingMode = false;
  }
}
 

@Override
public void activate(OperatorContext context)
{
  long largestRecoveryWindow = windowManager.getLargestCompletedWindow();
  if (largestRecoveryWindow == Stateless.WINDOW_ID
      || context.getValue(Context.OperatorContext.ACTIVATION_WINDOW_ID) > largestRecoveryWindow) {
    initializePreparedStatement();
    schedulePollTask();
  }
}
 

/**
 * Read available checkpoints from storage agent for all operators.
 * @param startTime
 * @param currentTime
 * @throws IOException
 */
public void syncCheckpoints(long startTime, long currentTime) throws IOException
{
  for (PTOperator oper : getAllOperators().values()) {
    StorageAgent sa = oper.operatorMeta.getValue(OperatorContext.STORAGE_AGENT);
    long[] windowIds = sa.getWindowIds(oper.getId());
    Arrays.sort(windowIds);
    oper.checkpoints.clear();
    for (long wid : windowIds) {
      if (wid != Stateless.WINDOW_ID) {
        oper.addCheckpoint(wid, startTime);
      }
    }
  }
}
 

/**
 * Initializes cassandra cluster connections as specified by the Connection State manager.
 * Aspects that are initialized here include Identifying primary key column names, non-primary key columns,
 * collection type columns, counter columns
 * It also queries the Keyspace and Table metadata for extracting the above information.
 * It finally prepares all possible prepared statements that can be executed in the lifetime of the operator
 * for various permutations like APPEND/REMOVE COLLECTIONS , LIST APPEND/PREPEND , set nulls, set TTLs etc
 * @param context The apex framework context
 */
@Override
public void activate(Context.OperatorContext context)
{
  ConnectionStateManager.ConnectionBuilder connectionBuilder = withConnectionBuilder();
  if (connectionBuilder == null) {
    connectionBuilder = buildConnectionBuilderFromPropertiesFile();
  }
  checkNotNull(connectionBuilder, " Connection Builder cannot be null.");
  connectionStateManager = connectionBuilder.initialize();
  cluster = connectionStateManager.getCluster();
  session = connectionStateManager.getSession();
  checkNotNull(session, "Cassandra session cannot be null");
  tuplePayloadClass = getPayloadPojoClass();
  columnDefinitions = new HashMap<>();
  counterColumns = new HashSet<>();
  collectionColumns = new HashSet<>();
  pkColumnNames = new HashSet<>();
  listColumns = new HashSet<>();
  mapColumns = new HashSet<>();
  setColumns = new HashSet<>();
  codecsForCassandraColumnNames = new HashMap<>();
  userDefinedTypeColumns = new HashSet<>();
  regularColumns = new HashSet<>();
  colNamesMap = new HashMap<>();
  getters = new HashMap<>();
  userDefinedTypesClass = new HashMap<>();
  uniqueHostsWrittenToInCurrentWindow = new HashSet<>();
  registerCodecs();
  KeyspaceMetadata keyspaceMetadata = cluster.getMetadata()
      .getKeyspace(connectionStateManager.getKeyspaceName());
  TableMetadata tableMetadata = keyspaceMetadata
      .getTable(connectionStateManager.getTableName());
  registerPrimaryKeyColumnDefinitions(tableMetadata);
  registerNonPKColumnDefinitions(tableMetadata);
  preparedStatementTypes = new HashMap<>();
  generatePreparedStatements();
  registerGettersForPayload();
  isInSafeMode = false;
  isInReconcilingMode = false;
  reconcilingWindowId = Stateless.WINDOW_ID;
  if ( (context.getValue(Context.OperatorContext.ACTIVATION_WINDOW_ID) != Stateless.WINDOW_ID) &&
      context.getValue(Context.OperatorContext.ACTIVATION_WINDOW_ID) <
      windowDataManager.getLargestCompletedWindow()) {
    isInSafeMode = true;
    reconcilingWindowId = windowDataManager.getLargestCompletedWindow() + 1;
    isInReconcilingMode = false;
  }
}
 

public long getLargestCompletedWindow()
{
  return Stateless.WINDOW_ID;
}
 

private long findLargestCompletedWindow(FSWindowReplayWAL wal, Long ceilingWindow) throws IOException
{
  if (!wal.fileDescriptors.isEmpty()) {
    FileSystemWAL.FileSystemWALReader reader = wal.getReader();

    //to find the largest window, we only need to look at the last file.
    NavigableSet<Integer> descendingParts = new TreeSet<>(wal.fileDescriptors.keySet()).descendingSet();
    for (int part : descendingParts) {
      FSWindowReplayWAL.FileDescriptor last = wal.fileDescriptors.get(part).last();
      reader.seek(new FileSystemWAL.FileSystemWALPointer(last.part, 0));

      long endOffset = -1;

      long lastWindow = Stateless.WINDOW_ID;
      Slice slice = readNext(reader);

      while (slice != null) {
        boolean skipComplete = skipNext(reader); //skip the artifact because we need just the largest window id.
        if (!skipComplete) {
          //artifact not saved so this window was not finished.
          break;
        }
        long offset = reader.getCurrentPointer().getOffset();

        long window = Longs.fromByteArray(slice.toByteArray());
        if (ceilingWindow != null && window > ceilingWindow) {
          break;
        }
        endOffset = offset;
        lastWindow = window;
        slice = readNext(reader);  //either null or next window
      }

      if (endOffset != -1) {
        wal.walEndPointerAfterRecovery = new FileSystemWAL.FileSystemWALPointer(last.part, endOffset);
        wal.windowWalParts.put(lastWindow, wal.walEndPointerAfterRecovery.getPartNum());
        return lastWindow;
      }
    }
  }
  return Stateless.WINDOW_ID;
}
 

@Override
public void setup(OperatorContext context)
{
  operatorContext = context;
  fileAccess.init();

  if (timeBucketAssigner == null) {
    // set default time bucket assigner
    MovingBoundaryTimeBucketAssigner movingBoundaryTimeBucketAssigner = new MovingBoundaryTimeBucketAssigner();
    setTimeBucketAssigner(movingBoundaryTimeBucketAssigner);
  }
  timeBucketAssigner.setPurgeListener(this);

  //setup all the managed state components
  timeBucketAssigner.setup(this);
  checkpointManager.setup(this);
  bucketsFileSystem.setup(this);

  if (buckets == null) {
    //create buckets map only once at start if it is not created.
    numBuckets = getNumBuckets();
    buckets = new HashMap<>();
  }
  for (Bucket bucket : buckets.values()) {
    if (bucket != null) {
      bucket.setup(this);
    }
  }

  stateTracker.setup(this);
  long activationWindow = context.getValue(OperatorContext.ACTIVATION_WINDOW_ID);

  if (activationWindow != Stateless.WINDOW_ID) {
    //All the wal files with windows <= activationWindow are loaded and kept separately as recovered data.
    try {

      Map<Long, Object> statePerWindow = checkpointManager.retrieveAllWindows();
      for (Map.Entry<Long, Object> stateEntry : statePerWindow.entrySet()) {
        Preconditions.checkArgument(stateEntry.getKey() <= activationWindow,
            stateEntry.getKey() + " greater than " + activationWindow);
        @SuppressWarnings("unchecked")
        Map<Long, Map<Slice, Bucket.BucketedValue>> state = (Map<Long, Map<Slice, Bucket.BucketedValue>>)
            stateEntry.getValue();
        if (state != null && !state.isEmpty()) {
          for (Map.Entry<Long, Map<Slice, Bucket.BucketedValue>> bucketEntry : state.entrySet()) {
            long bucketIdx = prepareBucket(bucketEntry.getKey());
            buckets.get(bucketIdx).recoveredData(stateEntry.getKey(), bucketEntry.getValue());
          }
        }
        // Skip write to WAL during recovery during replay from WAL.
        // Data only needs to be transferred to bucket data files.
        checkpointManager.save(state, stateEntry.getKey(), true /*skipWritingToWindowFile*/);
      }
    } catch (IOException e) {
      throw new RuntimeException("recovering", e);
    }
  }

  readerService = Executors.newFixedThreadPool(numReaders, new NameableThreadFactory("managedStateReaders"));
}
 

public FSIdempotentStorageManager()
{
  replayState = TreeMultimap.create();
  largestRecoveryWindow = Stateless.WINDOW_ID;
  recoveryPath = DEF_RECOVERY_PATH;
}
 

@Override
public long getLargestRecoveryWindow()
{
  return Stateless.WINDOW_ID;
}