com.datatorrent.api.annotation.Stateless Java Examples

private void initCheckpoint(PTOperator oper, Operator oo, Checkpoint checkpoint)
{
  try {
    LOG.debug("Writing activation checkpoint {} {} {}", checkpoint, oper, oo);
    long windowId = oper.isOperatorStateLess() ? Stateless.WINDOW_ID : checkpoint.windowId;
    StorageAgent agent = oper.operatorMeta.getValue(OperatorContext.STORAGE_AGENT);
    agent.save(oo, oper.id, windowId);
    if (agent instanceof AsyncStorageAgent) {
      ((AsyncStorageAgent)agent).flush(oper.id, windowId);
    }
  } catch (IOException e) {
    // inconsistent state, no recovery option, requires shutdown
    throw new IllegalStateException("Failed to write operator state after partition change " + oper, e);
  }
  oper.setRecoveryCheckpoint(checkpoint);
  if (!Checkpoint.INITIAL_CHECKPOINT.equals(checkpoint)) {
    oper.checkpoints.add(checkpoint);
  }
}
 

/**
 * This method checkpoints the given writer.
 * @param writer The writer to checkpoint.
 * @return new writer.
 */
public static AbstractFileOutputOperator checkpoint(AbstractFileOutputOperator writer, long windowId)
{
  if (windowId >= Stateless.WINDOW_ID) {
    writer.beforeCheckpoint(windowId);
  }
  Kryo kryo = new Kryo();
  ByteArrayOutputStream bos = new ByteArrayOutputStream();
  Output loutput = new Output(bos);
  kryo.writeObject(loutput, writer);
  loutput.close();

  Input lInput = new Input(bos.toByteArray());
  @SuppressWarnings("unchecked")
  AbstractFileOutputOperator checkPointedWriter = kryo.readObject(lInput, writer.getClass());
  lInput.close();

  return checkPointedWriter;
}
 

@Override
public void setup(Context.OperatorContext context)
{
  currentWindowRecoveryState = Lists.newLinkedList();
  if (referenceTimes == null) {
    referenceTimes = new ConcurrentHashMap<>();
  }
  scanner.setup(context);
  windowDataManager.setup(context);
  super.setup(context);

  long largestRecoveryWindow = windowDataManager.getLargestCompletedWindow();
  if (largestRecoveryWindow == Stateless.WINDOW_ID || context.getValue(Context.OperatorContext.ACTIVATION_WINDOW_ID) >
      largestRecoveryWindow) {
    scanner.startScanning(Collections.unmodifiableMap(referenceTimes));
  }
}
 

@Override
public void beginWindow(long windowId)
{
  super.beginWindow(windowId);
  currentWindowId = windowId;
  if ( currentWindowId != Stateless.WINDOW_ID) { // if it is not the first window of the application
    if (currentWindowId > reconcilingWindowId) {
      isInReplayMode = false;
      isInReconcilingMode = false;
    }
    if (currentWindowId == reconcilingWindowId) {
      isInReconcilingMode = true;
      isInReplayMode = false;
    }
    if (currentWindowId < reconcilingWindowId) {
      isInReconcilingMode = false;
      isInReplayMode = true;
    }
  }
  numDeletes = 0;
  numInserts = 0;
  numUpdates = 0;
  numUpserts = 0;
}
 

@Override
public void beginWindow(long windowId)
{
  currentWindowTupleCount = 0;
  currentWindowId = windowId;
  if ( currentWindowId != Stateless.WINDOW_ID) {
    if (currentWindowId > reconcilingPhaseWindowId) {
      isCurrentlyInSafeMode = false;
      isCurrentlyInReconcilingMode = false;
    }
    if (currentWindowId == reconcilingPhaseWindowId) {
      isCurrentlyInReconcilingMode = true;
      isCurrentlyInSafeMode = false;
    }
    if (currentWindowId < reconcilingPhaseWindowId) {
      isCurrentlyInReconcilingMode = false;
      isCurrentlyInSafeMode = true;
    }
  }
  LOG.info(" Current processing mode states Safe Mode = " + isCurrentlyInSafeMode + " Reconciling mode = "
      + isCurrentlyInReconcilingMode);
  LOG.info(" Current window ID = " + currentWindowId + " reconciling window ID = " + reconcilingPhaseWindowId);
}
 

/**
 *
 * @param id the value of id
 * @param name name of the operator
 * @param attributes the value of attributes
 * @param parentContext
 */
public OperatorContext(int id, @NotNull String name, AttributeMap attributes, Context parentContext)
{
  super(attributes, parentContext);
  this.lastProcessedWindowId = Stateless.WINDOW_ID;
  this.id = id;
  this.name = Preconditions.checkNotNull(name, "operator name");
  this.stateless = super.getValue(OperatorContext.STATELESS);
}
 

private void testMultiRollingFileFailedWriteOverwriteHelperCache1(EvenOddHDFSExactlyOnceWriter writer)
{
  File meta = new File(testMeta.getDir());
  writer.setFilePath(meta.getAbsolutePath());
  writer.setMaxLength(4);
  writer.setAlwaysWriteToTmp(testMeta.writeToTmp);
  writer.setup(testMeta.testOperatorContext);

  writer.beginWindow(0);
  writer.input.put(0);
  writer.input.put(1);
  writer.endWindow();

  AbstractFileOutputOperator checkPointWriter = checkpoint(writer, Stateless.WINDOW_ID);

  writer.beginWindow(1);
  writer.input.put(2);
  writer.input.put(3);
  writer.teardown();

  restoreCheckPoint(checkPointWriter, writer);
  writer.setup(testMeta.testOperatorContext);

  writer.beginWindow(1);
  writer.input.put(4);
  writer.input.put(5);
  writer.endWindow();

  writer.beginWindow(2);
  writer.input.put(6);
  writer.input.put(7);
  writer.input.put(8);
  writer.input.put(9);
  writer.input.put(6);
  writer.input.put(7);
  writer.input.put(10);
  writer.input.put(11);
  writer.endWindow();
  writer.committed(2);
}
 

private void testSingleRollingFileFailedWriteHelper(SingleHDFSExactlyOnceWriter writer)
{
  writer.setMaxLength(4);
  writer.setFilePath(testMeta.getDir());
  writer.setAlwaysWriteToTmp(testMeta.writeToTmp);
  writer.setup(testMeta.testOperatorContext);

  writer.beginWindow(0);
  writer.input.put(0);
  writer.input.put(1);
  writer.input.put(2);
  writer.endWindow();

  AbstractFileOutputOperator checkPointWriter = checkpoint(writer, Stateless.WINDOW_ID);

  writer.beginWindow(1);
  writer.input.put(3);
  writer.input.put(4);

  writer.teardown();

  restoreCheckPoint(checkPointWriter, writer);
  writer.setup(testMeta.testOperatorContext);

  writer.beginWindow(1);
  writer.input.put(3);
  writer.input.put(4);
  writer.input.put(5);
  writer.endWindow();

  writer.beginWindow(2);
  writer.input.put(6);
  writer.input.put(7);
  writer.input.put(8);
  writer.endWindow();
  writer.committed(2);

  writer.teardown();
}
 

private void testMultiFileFailedWriteHelper(EvenOddHDFSExactlyOnceWriter writer)
{
  File meta = new File(testMeta.getDir());
  writer.setFilePath(meta.getAbsolutePath());
  writer.setAlwaysWriteToTmp(testMeta.writeToTmp);
  writer.setup(testMeta.testOperatorContext);

  writer.beginWindow(0);
  writer.input.put(0);
  writer.input.put(1);
  writer.input.put(2);
  writer.input.put(3);
  writer.requestFinalize(EVEN_FILE);
  writer.endWindow();

  AbstractFileOutputOperator checkPointWriter = checkpoint(writer, Stateless.WINDOW_ID);

  writer.beginWindow(1);
  writer.input.put(4);
  writer.input.put(5);
  writer.requestFinalize(ODD_FILE);
  writer.endWindow();
  writer.teardown();

  restoreCheckPoint(checkPointWriter, writer);
  writer.setup(testMeta.testOperatorContext);

  writer.beginWindow(2);
  writer.input.put(6);
  writer.input.put(7);
  writer.input.put(8);
  writer.input.put(9);
  writer.endWindow();
  writer.beforeCheckpoint(2);
  writer.committed(2);
}
 

private void testSingleFileFailedWriteHelper(SingleHDFSExactlyOnceWriter writer)
{
  File meta = new File(testMeta.getDir());
  writer.setFilePath(meta.getAbsolutePath());
  writer.setAlwaysWriteToTmp(testMeta.writeToTmp);
  writer.setup(testMeta.testOperatorContext);

  writer.beginWindow(0);
  writer.input.put(0);
  writer.requestFinalize(SINGLE_FILE);
  writer.input.put(1);
  writer.endWindow();

  AbstractFileOutputOperator checkPointWriter = checkpoint(writer, Stateless.WINDOW_ID);

  writer.beginWindow(1);
  writer.input.put(2);

  writer.teardown();

  restoreCheckPoint(checkPointWriter, writer);
  writer.setup(testMeta.testOperatorContext);

  writer.beginWindow(1);
  writer.input.put(4);
  writer.requestFinalize(SINGLE_FILE);
  writer.input.put(5);
  writer.endWindow();

  writer.beginWindow(2);
  writer.input.put(6);
  writer.input.put(7);
  writer.endWindow();

  writer.committed(2);
  writer.teardown();
}
 

@Test
public void testOperatorMidWindowRestore()
{
  oper.beginWindow(0);
  oper.input.process("0");
  oper.input.process("0");
  oper.endWindow();

  oper.beginWindow(1);
  oper.input.process("1");

  AbstractFileOutputOperator checkPoint = AbstractFileOutputOperatorTest.checkpoint(oper, Stateless.WINDOW_ID);

  oper.input.process("1");
  oper.teardown();

  AbstractFileOutputOperatorTest.restoreCheckPoint(checkPoint, oper);

  oper.setup(testOperatorContext);

  oper.input.process("1");
  oper.input.process("1");
  oper.endWindow();

  oper.beginWindow(2);
  oper.input.process("2");
  oper.input.process("2");
  oper.endWindow();

  oper.teardown();

  AbstractFileOutputOperatorTest.checkOutput(-1, testMeta.getDir() + "/" + "0", "0\n" + "0\n");

  AbstractFileOutputOperatorTest.checkOutput(-1, testMeta.getDir() + "/" + "1", "1\n" + "1\n" + "1\n");

  AbstractFileOutputOperatorTest.checkOutput(-1, testMeta.getDir() + "/" + "2", "2\n" + "2\n");
}
 

@Test
public void testOperator()
{
  oper.beginWindow(0);
  oper.input.process("window 0");
  oper.input.process("window 0");
  oper.endWindow();

  AbstractFileOutputOperator checkPoint = AbstractFileOutputOperatorTest.checkpoint(oper, Stateless.WINDOW_ID);

  oper.beginWindow(1);
  oper.input.process("window 1");
  oper.teardown();

  AbstractFileOutputOperatorTest.restoreCheckPoint(checkPoint, oper);

  oper.setup(testOperatorContext);

  oper.beginWindow(1);
  oper.input.process("window_new 1");
  oper.input.process("window_new 1");
  oper.endWindow();

  oper.beginWindow(2);
  oper.input.process("window_new 2");
  oper.input.process("window_new 2");
  oper.endWindow();

  oper.teardown();

  AbstractFileOutputOperatorTest.checkOutput(-1, testMeta.getDir() + "/" + "0", "window 0\n" + "window 0\n");

  AbstractFileOutputOperatorTest.checkOutput(-1, testMeta.getDir() + "/" + "1", "window_new 1\n" + "window_new 1\n");

  AbstractFileOutputOperatorTest.checkOutput(-1, testMeta.getDir() + "/" + "2", "window_new 2\n" + "window_new 2\n");
}
 

@Test
public void testLargestRecoveryWindow()
{
  Pair<Context.OperatorContext, FSWindowDataManager> pair = createManagerAndContextFor(1);
  pair.second.setup(pair.first);
  Assert.assertEquals("largest recovery", Stateless.WINDOW_ID, pair.second.getLargestCompletedWindow());
  pair.second.teardown();
}
 

@Override
public void activate(OperatorContext context)
{
  long largestRecoveryWindow = windowManager.getLargestCompletedWindow();
  if (largestRecoveryWindow == Stateless.WINDOW_ID
      || context.getValue(Context.OperatorContext.ACTIVATION_WINDOW_ID) > largestRecoveryWindow) {
    initializePreparedStatement();
    schedulePollTask();
  }
}
 

@Override
public void setup(Context.OperatorContext context)
{
  if (windowDataManager != null) {
    windowDataManager.setup(context);
  }
  try {
    buildColumnSchemaForTable();
  } catch (Exception e) {
    throw new RuntimeException("Error while trying to build the schema definition for the Kudu table " + tableName,
        e);
  }
  windowManagerDataForScans = new HashMap<>();
  optionsEnabledForCurrentQuery = new HashMap<>();
  initPartitioner();
  initBuffer();
  // Scanner can only be initialized after initializing the partitioner
  initScanner(); // note that this is not streaming any data yet. Only warming up the scanner ready to read data
  initCurrentState();
  isCurrentlyInSafeMode = false;
  isCurrentlyInReconcilingMode = false;
  reconcilingPhaseWindowId = Stateless.WINDOW_ID;
  if ( (context.getValue(Context.OperatorContext.ACTIVATION_WINDOW_ID) != Stateless.WINDOW_ID) &&
      context.getValue(Context.OperatorContext.ACTIVATION_WINDOW_ID) <
      windowDataManager.getLargestCompletedWindow()) {
    isCurrentlyInSafeMode = true;
    reconcilingPhaseWindowId = windowDataManager.getLargestCompletedWindow() + 1;
    LOG.info("Set reconciling window ID as " + reconcilingPhaseWindowId);
    isCurrentlyInReconcilingMode = false;
  }
}
 

private void deployNodes(List<OperatorDeployInfo> nodeList) throws IOException
{
  for (OperatorDeployInfo ndi : nodeList) {
    StorageAgent backupAgent = getValue(OperatorContext.STORAGE_AGENT, ndi);
    assert (backupAgent != null);

    Context parentContext;
    if (ndi instanceof UnifierDeployInfo) {
      OperatorContext unifiedOperatorContext = new OperatorContext(0, ndi.name,
          ((UnifierDeployInfo)ndi).operatorAttributes, containerContext);
      parentContext = new PortContext(ndi.inputs.get(0).contextAttributes, unifiedOperatorContext);
      massageUnifierDeployInfo(ndi);
    } else {
      parentContext = containerContext;
    }

    OperatorContext ctx = new OperatorContext(ndi.id, ndi.name, ndi.contextAttributes, parentContext);
    ctx.attributes.put(OperatorContext.ACTIVATION_WINDOW_ID, ndi.checkpoint.windowId);
    Node<?> node = reuseOpNodes.get(ndi.id);
    if (node == null) {
      logger.info("Restoring operator {} to checkpoint {} stateless={}.", ndi.id, Codec.getStringWindowId(ndi.checkpoint.windowId), ctx.stateless);
      node = Node.retrieveNode(backupAgent.load(ndi.id, ctx.stateless ? Stateless.WINDOW_ID : ndi.checkpoint.windowId), ctx, ndi.type);
    } else {
      logger.info("Reusing previous operator instance {}", ndi.id);
      node = Node.retrieveNode(node.operator, ctx, ndi.type);
      node.setReuseOperator(true);
      reuseOpNodes.remove(ndi.id);
    }
    node.currentWindowId = ndi.checkpoint.windowId;
    node.applicationWindowCount = ndi.checkpoint.applicationWindowCount;
    node.firstWindowMillis = firstWindowMillis;
    node.windowWidthMillis = windowWidthMillis;

    node.setId(ndi.id);
    nodes.put(ndi.id, node);
    logger.debug("Marking operator {} as deployed.", node);
  }
}
 

@Override
public void activate(Context.OperatorContext context)
{
  ApexKuduConnection.ApexKuduConnectionBuilder apexKuduConnectionBuilder = getKuduConnectionConfig();
  apexKuduConnection = apexKuduConnectionBuilder.build();
  checkNotNull(apexKuduConnection,"Kudu connection cannot be null");
  kuduTable = apexKuduConnection.getKuduTable();
  kuduSession = apexKuduConnection.getKuduSession();
  kuduClientHandle = apexKuduConnection.getKuduClient();
  checkNotNull(kuduTable,"Kudu Table cannot be null");
  checkNotNull(kuduSession, "Kudu session cannot be null");
  allColumnDefs = new HashMap();
  primaryKeyColumnNames = new HashSet<>();
  kuduColumnBasedGetters = new HashMap();
  buildGettersForPojoPayload();
  reconcilingWindowId = Stateless.WINDOW_ID;
  // The operator is working in a replay mode where the upstream buffer is re-streaming the tuples
  // Note that there are two windows that need special core. The window that is being replayed and the subsequent
  // window that might have resulted in a crash which we are referring as reconciling window
  if ( (context.getValue(Context.OperatorContext.ACTIVATION_WINDOW_ID) != Stateless.WINDOW_ID) &&
      context.getValue(Context.OperatorContext.ACTIVATION_WINDOW_ID) <
      windowDataManager.getLargestCompletedWindow()) {
    reconcilingWindowId = windowDataManager.getLargestCompletedWindow() + 1;
  }

  if ( (context.getValue(Context.OperatorContext.ACTIVATION_WINDOW_ID) != Stateless.WINDOW_ID) &&
      context.getValue(Context.OperatorContext.ACTIVATION_WINDOW_ID) ==
      windowDataManager.getLargestCompletedWindow()) {
    reconcilingWindowId = windowDataManager.getLargestCompletedWindow();
  }
}
 

@Override
public void activate(OperatorContext ctx)
{
  if (context.getValue(OperatorContext.ACTIVATION_WINDOW_ID) != Stateless.WINDOW_ID &&
      context.getValue(OperatorContext.ACTIVATION_WINDOW_ID) < windowDataManager.getLargestCompletedWindow()) {
    // If it is a replay state, don't start the consumer
    return;
  }
  // Don't start thread here!
  // # of kafka_consumer_threads depends on the type of kafka client and the message
  // metadata(topic/partition/replica) layout
  consumer.start();
}
 

public Operator loadOperator(PTOperator oper)
{
  try {
    LOG.debug("Loading state for {}", oper);
    return (Operator)oper.operatorMeta.getValue(OperatorContext.STORAGE_AGENT).load(oper.id, oper.isOperatorStateLess() ? Stateless.WINDOW_ID : oper.recoveryCheckpoint.windowId);
  } catch (IOException e) {
    throw new RuntimeException("Failed to read partition state for " + oper, e);
  }
}
 

/**
 * Read available checkpoints from storage agent for all operators.
 * @param startTime
 * @param currentTime
 * @throws IOException
 */
public void syncCheckpoints(long startTime, long currentTime) throws IOException
{
  for (PTOperator oper : getAllOperators().values()) {
    StorageAgent sa = oper.operatorMeta.getValue(OperatorContext.STORAGE_AGENT);
    long[] windowIds = sa.getWindowIds(oper.getId());
    Arrays.sort(windowIds);
    oper.checkpoints.clear();
    for (long wid : windowIds) {
      if (wid != Stateless.WINDOW_ID) {
        oper.addCheckpoint(wid, startTime);
      }
    }
  }
}
 

public boolean isOperatorStateLess()
{
  if (operatorMeta.getDAG().getValue(OperatorContext.STATELESS) || operatorMeta.getValue(OperatorContext.STATELESS)) {
    return true;
  }

  return operatorMeta.getOperator().getClass().isAnnotationPresent(Stateless.class);
}
 

/**
 * Primarily resets the per window counter metrics.
 * @param windowId The windowid as provided by the apex framework
 */
@Override
public void beginWindow(long windowId)
{
  super.beginWindow(windowId);
  totalIgnoresInThisWindow = 0;
  totalWriteTimeoutsInThisWindow = 0;
  totalWriteRetriesInThisWindow =  0;
  uniqueHostsWrittenToInCurrentWindow.clear();
  successfullWrites = 0;
  ignoredRequestsDuetoIfExistsCheck = 0;
  writesWithConsistencyOne = 0;
  writesWithConsistencyTwo = 0;
  writesWithConsistencyThree = 0;
  writesWithConsistencyAll = 0;
  writesWithConsistencyLocalOne = 0;
  writesWithConsistencyQuorum = 0;
  writesWithConsistencyLocalQuorum = 0;
  writeWithConsistencyLocalSerial = 0;
  writesWithConsistencyEachQuorum = 0;
  writesWithConsistencySerial = 0;
  writesWithConsistencyAny = 0;
  currentWindowId = windowId;
  if ( currentWindowId != Stateless.WINDOW_ID) {
    if (currentWindowId > reconcilingWindowId) {
      isInSafeMode = false;
      isInReconcilingMode = false;
    }
    if (currentWindowId == reconcilingWindowId) {
      isInReconcilingMode = true;
      isInSafeMode = false;
    }
    if (currentWindowId < reconcilingWindowId) {
      isInReconcilingMode = false;
      isInSafeMode = true;
    }
  }
}
 

/**
 * Implement ActivationListener Interface.
 */
@Override
public void activate(OperatorContext ctx)
{
  // If it is a replay state, don't start the consumer
  if (context.getValue(OperatorContext.ACTIVATION_WINDOW_ID) != Stateless.WINDOW_ID &&
      context.getValue(OperatorContext.ACTIVATION_WINDOW_ID) < windowDataManager.getLargestCompletedWindow()) {
    return;
  }
  consumer.start();
}
 

@Override
public long[] getWindowIds(int operatorId) throws IOException
{
  Path lPath = new Path(path + Path.SEPARATOR + String.valueOf(operatorId));
  try {
    FileStatus status = fileContext.getFileStatus(lPath);
    if (!status.isDirectory()) {
      throw new RuntimeException("Checkpoint location is not a directory");
    }
  } catch (FileNotFoundException ex) {
    // During initialization checkpoint directory may not exists.
    fileContext.mkdir(lPath, FsPermission.getDirDefault(), true);
  }

  RemoteIterator<FileStatus> fileStatusRemoteIterator = fileContext.listStatus(lPath);
  List<Long> lwindows = new ArrayList<>();
  while (fileStatusRemoteIterator.hasNext()) {
    FileStatus fileStatus = fileStatusRemoteIterator.next();
    String name = fileStatus.getPath().getName();
    if (name.equals(TMP_FILE)) {
      continue;
    }
    lwindows.add(STATELESS_CHECKPOINT_WINDOW_ID.equals(name) ? Stateless.WINDOW_ID : Long.parseLong(name, 16));
  }
  long[] windowIds = new long[lwindows.size()];
  for (int i = 0; i < windowIds.length; i++) {
    windowIds[i] = lwindows.get(i);
  }
  return windowIds;
}
 

private void singleFileMultiRollingFailureHelper(SingleHDFSExactlyOnceWriter writer)
{
  writer.setAlwaysWriteToTmp(testMeta.writeToTmp);
  writer.setup(testMeta.testOperatorContext);

  writer.beginWindow(0);
  writer.input.put(0);
  writer.input.put(1);
  writer.input.put(2);
  writer.endWindow();

  writer.beginWindow(1);
  writer.input.put(3);
  writer.input.put(4);

  AbstractFileOutputOperator checkPointWriter = checkpoint(writer, Stateless.WINDOW_ID);

  writer.input.put(3);
  writer.input.put(4);
  writer.input.put(5);
  writer.endWindow();

  writer.beginWindow(2);
  writer.input.put(6);
  writer.input.put(7);
  writer.input.put(8);
  writer.endWindow();
  writer.teardown();

  restoreCheckPoint(checkPointWriter, writer);
  writer.setup(testMeta.testOperatorContext);

  writer.beginWindow(1);
  writer.input.put(0);
  writer.input.put(1);
  writer.input.put(2);
  writer.endWindow();

  writer.beginWindow(2);
  writer.input.put(3);
  writer.input.put(4);
  writer.input.put(5);
  writer.input.put(6);
  writer.endWindow();
  writer.committed(2);
}
 

private void testMultiRollingFileFailedWriteOverwriteHelper(EvenOddHDFSExactlyOnceWriter writer) throws IOException
{
  String evenFileName = testMeta.getDir() + File.separator + EVEN_FILE;
  String oddFileName = testMeta.getDir() + File.separator + ODD_FILE;

  populateFile(EVEN_FILE + ".0", "0\n" + "2\n" + "4\n");
  populateFile(ODD_FILE + ".0", "1\n" + "3\n" + "5\n");

  File meta = new File(testMeta.getDir());
  writer.setFilePath(meta.getAbsolutePath());
  writer.setMaxLength(4);
  writer.setAlwaysWriteToTmp(testMeta.writeToTmp);
  writer.setup(testMeta.testOperatorContext);

  writer.beginWindow(0);
  writer.input.process(0);
  writer.input.process(1);
  writer.endWindow();

  AbstractFileOutputOperator checkPointWriter = checkpoint(writer, Stateless.WINDOW_ID);

  writer.beginWindow(1);
  writer.input.process(2);
  writer.input.process(3);
  writer.teardown();

  restoreCheckPoint(checkPointWriter, writer);
  writer.setup(testMeta.testOperatorContext);

  writer.beginWindow(1);
  writer.input.process(4);
  writer.input.process(5);
  writer.endWindow();

  writer.beginWindow(2);
  writer.input.process(6);
  writer.input.process(7);
  writer.input.process(8);
  writer.input.process(9);
  writer.input.process(6);
  writer.input.process(7);
  writer.input.process(10);
  writer.input.process(11);
  writer.endWindow();
  writer.committed(2);

  //Even file
  String correctContents = "0\n" + "4\n" + "6\n";
  checkOutput(0, evenFileName, correctContents);
  correctContents = "8\n" + "6\n" + "10\n";
  checkOutput(1, evenFileName, correctContents);

  //Odd file
  correctContents = "1\n" + "5\n" + "7\n";
  checkOutput(0, oddFileName, correctContents);
  correctContents = "9\n" + "7\n" + "11\n";
  checkOutput(1, oddFileName, correctContents);
}
 

private void testMultiRollingFileFailedWriteHelper(EvenOddHDFSExactlyOnceWriter writer)
{
  File meta = new File(testMeta.getDir());
  writer.setFilePath(meta.getAbsolutePath());
  writer.setMaxLength(4);
  writer.setAlwaysWriteToTmp(testMeta.writeToTmp);

  writer.setup(testMeta.testOperatorContext);

  writer.beginWindow(0);
  writer.input.put(0);
  writer.input.put(1);
  writer.endWindow();

  AbstractFileOutputOperator checkPointWriter = checkpoint(writer, Stateless.WINDOW_ID);

  writer.beginWindow(1);
  writer.input.put(2);
  writer.input.put(3);
  writer.teardown();

  restoreCheckPoint(checkPointWriter, writer);
  writer.setup(testMeta.testOperatorContext);

  writer.beginWindow(1);
  writer.input.put(2);
  writer.input.put(3);
  writer.endWindow();

  writer.beginWindow(2);
  writer.input.put(4);
  writer.input.put(5);
  writer.endWindow();

  writer.beginWindow(3);
  writer.input.put(6);
  writer.input.put(7);
  writer.input.put(8);
  writer.input.put(9);
  writer.input.put(6);
  writer.input.put(7);
  writer.endWindow();
  writer.committed(3);
}
 

@Test
public void testSingleRollingFileFailedWrite1()
{
  SingleHDFSExactlyOnceWriter writer = new SingleHDFSExactlyOnceWriter();
  writer.setFilePath(testMeta.getDir());
  writer.setMaxLength(4);
  writer.setAlwaysWriteToTmp(testMeta.writeToTmp);
  writer.setup(testMeta.testOperatorContext);

  writer.beginWindow(0);
  writer.input.put(0);
  writer.input.put(1);
  writer.input.put(2);
  writer.endWindow();

  writer.beginWindow(1);
  writer.input.put(3);
  writer.input.put(4);
  writer.endWindow();

  AbstractFileOutputOperator checkPointWriter = checkpoint(writer, Stateless.WINDOW_ID);
  AbstractFileOutputOperator checkPointWriter1 = checkpoint(writer, Stateless.WINDOW_ID);

  LOG.debug("Checkpoint endOffsets={}", checkPointWriter.endOffsets);

  writer.beginWindow(2);
  writer.input.put(5);
  writer.teardown();

  restoreCheckPoint(checkPointWriter, writer);
  LOG.debug("Checkpoint endOffsets={}", checkPointWriter.endOffsets);
  writer.setup(testMeta.testOperatorContext);

  writer.beginWindow(2);
  writer.input.put(5);
  writer.endWindow();

  writer.beginWindow(3);
  writer.input.put(6);
  writer.input.put(7);
  writer.input.put(8);
  writer.endWindow();

  writer.teardown();

  restoreCheckPoint(checkPointWriter1, writer);
  writer.setup(testMeta.testOperatorContext);
  writer.committed(2);

  String singleFilePath = testMeta.getDir() + File.separator + SINGLE_FILE;

  //Rolling file 0
  String correctContents = "0\n" + "1\n" + "2\n";
  checkOutput(0, singleFilePath, correctContents);

  //Rolling file 1
  correctContents = "3\n" + "4\n";
  checkOutput(1, singleFilePath, correctContents);
}
 

@Test
public void testLargestRecoveryWindow()
{
  Assert.assertEquals("largest recovery", Stateless.WINDOW_ID, testMeta.storageManager.getLargestRecoveryWindow());
}
 

@Test
public void testDeleteDoesNotRemoveTmpFiles() throws IOException
{
  Pair<Context.OperatorContext, FSWindowDataManager> pair1 = createManagerAndContextFor(1);
  pair1.second.setup(pair1.first);

  Pair<Context.OperatorContext, FSWindowDataManager> pair2 = createManagerAndContextFor(2);
  pair2.second.setup(pair2.first);

  Pair<Context.OperatorContext, FSWindowDataManager> pair3 = createManagerAndContextFor(3);
  pair3.second.setup(pair3.first);

  Map<Integer, String> dataOf1 = Maps.newHashMap();
  dataOf1.put(1, "one");
  dataOf1.put(2, "two");
  dataOf1.put(3, "three");

  Map<Integer, String> dataOf2 = Maps.newHashMap();
  dataOf2.put(4, "four");
  dataOf2.put(5, "five");
  dataOf2.put(6, "six");

  Map<Integer, String> dataOf3 = Maps.newHashMap();
  dataOf2.put(7, "seven");
  dataOf2.put(8, "eight");
  dataOf2.put(9, "nine");

  for (int i = 1; i <= 9; ++i) {
    pair1.second.save(dataOf1, i);
  }

  for (int i = 1; i <= 6; ++i) {
    pair2.second.save(dataOf2, i);
  }

  for (int i = 1; i <= 3; ++i) {
    pair3.second.save(dataOf3, i);
  }

  pair1.second.teardown();
  pair2.second.teardown();
  pair3.second.teardown();

  FSWindowDataManager fsManager = (FSWindowDataManager)pair1.second.partition(1, Sets.newHashSet(2, 3)).get(0);
  fsManager.setup(pair1.first);

  Assert.assertEquals("recovery window", 3, fsManager.getLargestCompletedWindow());

  Map<Integer, Object> artifacts = fsManager.retrieveAllPartitions(1);
  Assert.assertEquals("num artifacts", 3, artifacts.size());

  fsManager.committed(3);
  fsManager.teardown();

  testMeta.attributes.put(Context.OperatorContext.ACTIVATION_WINDOW_ID, 3L);
  fsManager.setup(pair1.first);
  Assert.assertEquals("recovery window", Stateless.WINDOW_ID, fsManager.getLargestCompletedWindow());
  fsManager.teardown();
}