Java Code Examples for com.datatorrent.api.Attribute#AttributeMap

@Before
public void setup() throws IOException
{
  final String windowDataDir = "target/" + this.getClass().getSimpleName();
  final File windowDataDirFile = new File(windowDataDir);
  if (windowDataDirFile.exists()) {
    FileUtils.deleteFile(windowDataDirFile, true);
  }

  Attribute.AttributeMap attributeMap = new Attribute.AttributeMap.DefaultAttributeMap();
  attributeMap.put(DAG.APPLICATION_PATH, windowDataDir);

  context = mockOperatorContext(12345, attributeMap);

  sink = new CollectorTestSink<>();
  builder = new MockSiteToSiteClient.Builder();
  windowDataManager = new FSWindowDataManager();

  operator = new NiFiSinglePortInputOperator(builder, windowDataManager);
  operator.outputPort.setSink(sink);
}
 

@Test
public void testApplication() throws Exception
{
  Configuration conf = new Configuration(false);
  conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
  EmbeddedAppLauncher<?> launcher = Launcher.getLauncher(Launcher.LaunchMode.EMBEDDED);
  Attribute.AttributeMap launchAttributes = new Attribute.AttributeMap.DefaultAttributeMap();
  launchAttributes.put(EmbeddedAppLauncher.RUN_ASYNC, true);
  SimpleTransformApplication simpleTransformApplication = new SimpleTransformApplication();
  simpleTransformApplication.outputFn = outputFn;
  Launcher.AppHandle appHandle = launcher.launchApp(simpleTransformApplication, conf, launchAttributes);
  int sleepTimeCounterForLoopExit = 0;
  int sleepTimePerIteration = 500;
  // wait until expected result count or timeout
  while (results.size() < simpleTransformApplication.pojoDataGenerator.getMaxTuples()) {
    sleepTimeCounterForLoopExit += sleepTimePerIteration;
    if (sleepTimeCounterForLoopExit > 30000) {
      break;
    }
    Thread.sleep(sleepTimePerIteration);
  }
  appHandle.shutdown(Launcher.ShutdownMode.KILL);
  assertEquals(results.size(), simpleTransformApplication.pojoDataGenerator.getMaxTuples());
  assertTransformationsGenerated(simpleTransformApplication);
}
 

@Override
protected void starting(Description description)
{
  super.starting(description);
  outputPath = new File(
      "target" + Path.SEPARATOR + description.getClassName() + Path.SEPARATOR + description.getMethodName())
          .getPath();

  Attribute.AttributeMap attributes = new Attribute.AttributeMap.DefaultAttributeMap();
  attributes.put(DAG.DAGContext.APPLICATION_ID, description.getClassName());
  attributes.put(DAG.DAGContext.APPLICATION_PATH, outputPath);
  context = mockOperatorContext(1, attributes);

  underTest = new S3Reconciler();
  underTest.setAccessKey("");
  underTest.setSecretKey("");

  underTest.setup(context);

  MockitoAnnotations.initMocks(this);
  PutObjectResult result = new PutObjectResult();
  result.setETag(outputPath);
  when(s3clientMock.putObject((PutObjectRequest)any())).thenReturn(result);
  underTest.setS3client(s3clientMock);
}
 

@Before
public void setup() throws IOException
{
  final String windowDataDir = "target/" + this.getClass().getSimpleName();
  final File windowDataDirFile = new File(windowDataDir);
  if (windowDataDirFile.exists()) {
    FileUtils.deleteFile(windowDataDirFile, true);
  }

  Attribute.AttributeMap attributeMap = new Attribute.AttributeMap.DefaultAttributeMap();
  attributeMap.put(DAG.APPLICATION_PATH, windowDataDir);

  context = mockOperatorContext(12345, attributeMap);

  windowDataManager = new FSWindowDataManager();

  stsBuilder = new MockSiteToSiteClient.Builder();
  dpBuilder = new StringNiFiDataPacketBuilder();
  operator = new NiFiSinglePortOutputOperator(stsBuilder, dpBuilder, windowDataManager, 1);
}
 

@Override
protected void starting(Description description)
{
  super.starting(description);
  outputPath = new File("target/" + description.getClassName() + "/" + description.getMethodName()).getPath();

  Attribute.AttributeMap attributes = new Attribute.AttributeMap.DefaultAttributeMap();
  attributes.put(DAG.DAGContext.APPLICATION_ID, description.getClassName());
  attributes.put(DAG.DAGContext.APPLICATION_PATH, outputPath);
  context = mockOperatorContext(1, attributes);

  underTest = new FSRecordCompactionOperator<byte[]>();
  underTest.setConverter(new GenericFileOutputOperator.NoOpConverter());
  underTest.setup(context);
  underTest.setMaxIdleWindows(10);
}
 

private FinalVars(LogicalPlan dag, long tms)
{
  Attribute.AttributeMap attributes = dag.getAttributes();
  /* try to align to it to please eyes. */
  windowStartMillis = tms - (tms % 1000);

  if (attributes.get(LogicalPlan.APPLICATION_PATH) == null) {
    throw new IllegalArgumentException("Not set: " + LogicalPlan.APPLICATION_PATH);
  }

  this.appPath = attributes.get(LogicalPlan.APPLICATION_PATH);

  if (attributes.get(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS) == null) {
    attributes.put(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS, 500);
  }
  if (attributes.get(LogicalPlan.CHECKPOINT_WINDOW_COUNT) == null) {
    attributes.put(LogicalPlan.CHECKPOINT_WINDOW_COUNT, 30000 / attributes.get(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS));
  }

  this.heartbeatTimeoutMillis = dag.getValue(LogicalPlan.HEARTBEAT_TIMEOUT_MILLIS);
  this.maxWindowsBehindForStats = dag.getValue(LogicalPlan.STATS_MAX_ALLOWABLE_WINDOWS_LAG);
  this.enableStatsRecording = dag.getValue(LogicalPlan.ENABLE_STATS_RECORDING);
  this.rpcLatencyCompensationSamples = dag.getValue(LogicalPlan.RPC_LATENCY_COMPENSATION_SAMPLES);
}
 

/**
 * Shortcut to run an application with the modified configuration.
 *
 * @param app           - Application to be run
 * @param configuration - Configuration
 * @param runMillis     - The time after which the application will be shutdown; pass 0 to run indefinitely.
 */
public static void runApp(StreamingApplication app, Configuration configuration, int runMillis)
{
  EmbeddedAppLauncher launcher = newInstance();
  Attribute.AttributeMap launchAttributes = new Attribute.AttributeMap.DefaultAttributeMap();
  launchAttributes.put(RUN_MILLIS, (long)runMillis);
  launcher.launchApp(app, configuration, launchAttributes);
}
 

@Override
protected void starting(org.junit.runner.Description description)
{
  String methodName = description.getMethodName();
  String className = description.getClassName();
  this.dir = "target/" + className + "/" + methodName;
  Attribute.AttributeMap operAttributes = new Attribute.AttributeMap.DefaultAttributeMap();
  operAttributes.put(Context.DAGContext.APPLICATION_PATH, dir);
  Attribute.AttributeMap portAttributes = new Attribute.AttributeMap.DefaultAttributeMap();
  portAttributes.put(Context.PortContext.TUPLE_CLASS, EventRecord.class);
  context = mockOperatorContext(1, operAttributes);
  portContext = new TestPortContext(portAttributes);
}
 

@Override
protected void starting(org.junit.runner.Description description)
{
  String methodName = description.getMethodName();
  String className = description.getClassName();
  this.dir = "target/" + className + "/" + methodName;
  Attribute.AttributeMap attributes = new Attribute.AttributeMap.DefaultAttributeMap();
  attributes.put(Context.DAGContext.APPLICATION_PATH, dir);
  context = mockOperatorContext(1, attributes);
  Attribute.AttributeMap portAttributes = new Attribute.AttributeMap.DefaultAttributeMap();
  portAttributes.put(Context.PortContext.TUPLE_CLASS, SimpleOrder.class);
  portContext = new TestPortContext(portAttributes);
}
 

public Attribute.AttributeMap getApplicationAttributes()
{
  LogicalPlan lp = getLogicalPlan();
  try {
    return lp.getAttributes().clone();
  } catch (CloneNotSupportedException ex) {
    throw new RuntimeException("Cannot clone DAG attributes", ex);
  }
}
 

@Override
public YarnAppHandleImpl launchApp(final StreamingApplication app, Configuration conf, Attribute.AttributeMap launchParameters) throws LauncherException
{
  if (launchParameters != null) {
    for (Map.Entry<Attribute<?>, Object> entry : launchParameters.entrySet()) {
      String property = propMapping.get(entry.getKey());
      if (property != null) {
        setConfiguration(conf, property, entry.getValue());
      }
    }
  }
  try {
    String name = app.getClass().getName();
    StramAppLauncher appLauncher = new StramAppLauncher(name, conf);
    appLauncher.loadDependencies();
    StreamingAppFactory appFactory = new StreamingAppFactory(name, app.getClass())
    {
      @Override
      public LogicalPlan createApp(LogicalPlanConfiguration planConfig)
      {
        return super.createApp(app, planConfig);
      }
    };
    ApplicationId appId = appLauncher.launchApp(appFactory);
    appLauncher.resetContextClassLoader();
    return new YarnAppHandleImpl(appId, conf);
  } catch (Exception ex) {
    throw new LauncherException(ex);
  }
}
 

@Override
protected void starting(Description description)
{
  super.starting(description);
  outputPath = new File("target/" + description.getClassName() + "/" + description.getMethodName()).getPath();

  oper = new FileStitcher<OutputFileMetadata>();
  oper.setFilePath(outputPath);
  String appDirectory = outputPath;

  Attribute.AttributeMap attributes = new Attribute.AttributeMap.DefaultAttributeMap();
  attributes.put(DAG.DAGContext.APPLICATION_ID, description.getClassName());
  attributes.put(DAG.DAGContext.APPLICATION_PATH, appDirectory);
  context = mockOperatorContext(1, attributes);

  oper.setup(context);

  try {
    File outDir = new File(outputPath);
    FileUtils.forceMkdir(outDir);

    blocksDir = new File(context.getValue(Context.DAGContext.APPLICATION_PATH), BlockWriter.DEFAULT_BLOCKS_DIR);
    blocksDir.mkdirs();

    long blockID = 1000;

    for (int i = 0; i < FILE_CONTENTS.length; i++) {
      List<Long> blockIDs = Lists.newArrayList();

      File file = new File(outputPath, i + ".txt");

      FileUtils.write(file, FILE_CONTENTS[i]);

      int offset = 0;
      for (; offset < FILE_CONTENTS[i].length(); offset += BLOCK_SIZE, blockID++) {
        String blockContents;
        if (offset + BLOCK_SIZE < FILE_CONTENTS[i].length()) {
          blockContents = FILE_CONTENTS[i].substring(offset, offset + BLOCK_SIZE);
        } else {
          blockContents = FILE_CONTENTS[i].substring(offset);
        }
        FileUtils.write(new File(blocksDir, blockID + ""), blockContents);
        blockIDs.add(blockID);
      }

      FileMetadata fileMetadata = new FileMetadata(file.getPath());
      fileMetadata.setBlockIds(ArrayUtils.toPrimitive(blockIDs.toArray(new Long[0])));
      fileMetadataList.add(fileMetadata);
    }
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}
 

@Override
public Attribute.AttributeMap getAttributes()
{
  return null;
}
 

ClusterAppContextImpl(Attribute.AttributeMap attributes)
{
  super(attributes, null);
}
 

@Override
protected void starting(Description description)
{
  final String methodName = description.getMethodName();
  final String className = description.getClassName();

  testBase = new SQSTestBase();
  if (testBase.validateTestCreds() == false) {
    return;
  }
  testBase.generateCurrentQueueName(methodName);
  try {
    testBase.beforTest();
  } catch (AssumptionViolatedException ave) {
    throw ave;
  } catch (Exception e) {
    throw new RuntimeException(e);
  }

  baseDir = "target/" + className + "/" + methodName;

  Attribute.AttributeMap attributeMap = new Attribute.AttributeMap.DefaultAttributeMap();
  attributeMap.put(Context.OperatorContext.SPIN_MILLIS, 500);
  attributeMap.put(Context.DAGContext.APPLICATION_PATH, baseDir);

  context = mockOperatorContext(1, attributeMap);
  operator = new JMSStringInputOperator();
  operator.setConnectionFactoryBuilder(new JMSBase.ConnectionFactoryBuilder()
  {

    @Override
    public ConnectionFactory buildConnectionFactory()
    {
      // Create the connection factory using the environment variable credential provider.
      // Connections this factory creates can talk to the queues in us-east-1 region.
      SQSConnectionFactory connectionFactory =
          SQSConnectionFactory.builder()
          .withRegion(Region.getRegion(Regions.US_EAST_1))
          .withAWSCredentialsProvider(new PropertiesFileCredentialsProvider(testBase.getDevCredsFilePath()))
          .build();
      return connectionFactory;
    }

    @Override
    public String toString()
    {
      return className + "/" + methodName + "/ConnectionFactoryBuilder";
    }

  });
  operator.setSubject(testBase.getCurrentQueueName());
  // for SQS ack mode should be "AUTO_ACKNOWLEDGE" and transacted = false
  operator.setAckMode("AUTO_ACKNOWLEDGE");
  operator.setTransacted(false);

  sink = new CollectorTestSink<>();
  operator.output.setSink(sink);
  operator.setup(context);
  operator.activate(context);
}
 

public TestPortContext(@NotNull Attribute.AttributeMap attributeMap)
{
  this.attributeMap = Preconditions.checkNotNull(attributeMap, "attributes");
}
 

@Override
public Attribute.AttributeMap getAttributes()
{
  return attributeMap;
}
 

@Test
public void testRecoveryAndIdempotency() throws Exception
{
  RabbitMQInputOperator operator = new RabbitMQInputOperator();
  operator.setWindowDataManager(new FSWindowDataManager());
  operator.setHost("localhost");
  operator.setExchange("testEx");
  operator.setExchangeType("fanout");

  Attribute.AttributeMap attributeMap = new Attribute.AttributeMap.DefaultAttributeMap();
  CollectorTestSink<Object> sink = new CollectorTestSink<Object>();

  operator.outputPort.setSink(sink);
  OperatorContext context = mockOperatorContext(1, attributeMap);

  operator.setup(context);
  operator.activate(context);

  final RabbitMQMessageGenerator publisher = new RabbitMQMessageGenerator();
  publisher.setup();
  publisher.generateMessages(5);

  Thread.sleep(10000);

  operator.beginWindow(1);
  operator.emitTuples();
  operator.endWindow();

  operator.deactivate();
  Assert.assertEquals("num of messages in window 1", 15, sink.collectedTuples.size());

  // failure and then re-deployment of operator
  sink.collectedTuples.clear();
  operator.setup(context);
  operator.activate(context);

  Assert.assertEquals("largest recovery window", 1, operator.getWindowDataManager().getLargestCompletedWindow());
  operator.beginWindow(1);
  operator.endWindow();
  Assert.assertEquals("num of messages in window 1", 15, sink.collectedTuples.size());
  sink.collectedTuples.clear();

  operator.deactivate();
  operator.teardown();
  operator.getWindowDataManager().committed(1);
  publisher.teardown();
}
 

@Test
public void testApplication() throws Exception
{
  File targetDir = new File(TARGET_DIR);
  FileUtils.deleteDirectory(targetDir);
  FileUtils.forceMkdir(targetDir);

  KafkaUnit ku = kafkaUnitRule.getKafkaUnit();
  String topicName = "testTopic";
  // topic creation is async and the producer may also auto-create it
  ku.createTopic(topicName, 1);

  // produce test data
  String[] words = "count count the words from kafka and store them in a file".split("\\s+");
  for (String word : words) {
    ku.sendMessages(new KeyedMessage<String, String>(topicName, word));
  }

  Configuration conf = new Configuration(false);
  conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
  conf.set("apex.operator.kafkaInput.prop.topics", topicName);
  conf.set("apex.operator.kafkaInput.prop.clusters", "localhost:" + brokerPort);
  conf.set("apex.operator.kafkaInput.prop.maxTuplesPerWindow", "2"); // consume one word per window
  conf.set("apex.operator.kafkaInput.prop.initialOffset", "EARLIEST");
  conf.set("apex.operator.fileWriter.prop.filePath", TARGET_DIR);

  EmbeddedAppLauncher<?> launcher = Launcher.getLauncher(LaunchMode.EMBEDDED);
  Attribute.AttributeMap launchAttributes = new Attribute.AttributeMap.DefaultAttributeMap();
  launchAttributes.put(EmbeddedAppLauncher.RUN_ASYNC, true); // terminate after results are available
  AppHandle appHandle = launcher.launchApp(new ExactlyOnceFileOutputApp(), conf, launchAttributes);

  long timeout = System.currentTimeMillis() + 60000; // 60s timeout

  File outputFile = new File(TARGET_DIR, ExactlyOnceFileOutputApp.FileWriter.FILE_NAME_PREFIX);
  while (!outputFile.exists() && timeout > System.currentTimeMillis()) {
    Thread.sleep(1000);
    LOG.debug("Waiting for {}", outputFile);
  }

  Assert.assertTrue("output file exists " + ExactlyOnceFileOutputApp.FileWriter.FILE_NAME_PREFIX, outputFile.exists() &&
      outputFile.isFile());

  String result = FileUtils.readFileToString(outputFile);
  Assert.assertTrue(result.contains("count=2"));

  appHandle.shutdown(ShutdownMode.KILL);
}
 

/**
 * Launch application with configuration and launch parameters.
 *
 * Launch the given streaming application with the given configuration and parameters. The parameters should be from
 * the list of parameters supported by the launcher. To find out more about the supported parameters look at the
 * documentation of the individual launcher.
 *
 * @param application  - Application to be run
 * @param configuration - Application Configuration
 * @param launchParameters - Launch Parameters
 *
 * @return The application handle
 */
public abstract H launchApp(StreamingApplication application, Configuration configuration, Attribute.AttributeMap launchParameters) throws LauncherException;