com.datatorrent.netlet.util.DTThrowable Java Examples

/**
 * Create connection with database.
 */
@Override
public void connect()
{
  try {
    if (cluster == null) {
      buildCluster();
    }
    session = cluster.connect();
    logger.debug("Cassandra connection Success");
  } catch (DriverException ex) {
    throw new RuntimeException("closing database resource", ex);
  } catch (Throwable t) {
    DTThrowable.rethrow(t);
  }
}
 

public PercolateResponse percolate(String[] indexNames, String documentType, Object tuple)
{
  XContentBuilder docBuilder;
  try {

    docBuilder = XContentFactory.jsonBuilder().startObject();
    docBuilder.field("doc").startObject(); //This is needed to designate the document
    docBuilder.field("content", tuple);
    docBuilder.endObject(); //End of the doc field
    docBuilder.endObject();//End of the JSON root object

    return client.preparePercolate().setIndices(indexNames)
        .setDocumentType(documentType)
        .setSource(docBuilder)
        .execute().actionGet();
  } catch (IOException e) {
    DTThrowable.rethrow(e);
  }
  return null;
}
 

@Override
public void run()
{
  running = true;
  try {
    while (running) {
      if (trigger || (System.currentTimeMillis() - scanIntervalMillis >= lastScanMillis)) {
        trigger = false;
        for (String afile : files) {
          scan(new Path(afile), null);
        }
        scanComplete();
      } else {
        Thread.sleep(sleepMillis);
      }
    }
  } catch (Throwable throwable) {
    LOG.error("service", throwable);
    running = false;
    atomicThrowable.set(throwable);
    DTThrowable.rethrow(throwable);
  }
}
 

@Override
public void setup(OperatorContext context)
{
  // Needed to setup idempotency storage manager in setter
  this.context = context;
  this.operatorContextId = context.getId();

  try {
    connFactory.setHost("localhost");
    connection = connFactory.newConnection();
    channel = connection.createChannel();
    channel.exchangeDeclare(exchange, "fanout");

    this.windowDataManager.setup(context);

  } catch (IOException ex) {
    logger.debug(ex.toString());
    DTThrowable.rethrow(ex);
  }
}
 

@Override
public void storeCommittedWindowId(String appId, int operatorId,long windowId)
{
  byte[] WindowIdBytes = toBytes(windowId);
  String columnKey = appId + "_" + operatorId + "_" + lastWindowColumnName;
  lastWindowColumnBytes = columnKey.getBytes();
  Mutation mutation = new Mutation(rowBytes);
  mutation.put(columnFamilyBytes, lastWindowColumnBytes, WindowIdBytes);
  try {
    batchwriter.addMutation(mutation);
    batchwriter.flush();
  } catch (MutationsRejectedException e) {
    logger.error("error getting committed window id", e);
    DTThrowable.rethrow(e);
  }
}
 

@Override
public long getCommittedWindowId(String appId, int operatorId)
{
  byte[] value = null;
  Authorizations auths = new Authorizations();
  Scanner scan = null;
  String columnKey = appId + "_" + operatorId + "_" + lastWindowColumnName;
  lastWindowColumnBytes = columnKey.getBytes();
  try {
    scan = connector.createScanner(tableName, auths);
  } catch (TableNotFoundException e) {
    logger.error("error getting committed window id", e);
    DTThrowable.rethrow(e);
  }
  scan.setRange(new Range(new Text(rowBytes)));
  scan.fetchColumn(new Text(columnFamilyBytes), new Text(lastWindowColumnBytes));
  for (Entry<Key, Value> entry : scan) {
    value = entry.getValue().get();
  }
  if (value != null) {
    long longval = toLong(value);
    return longval;
  }
  return -1;
}
 

/**
 * Creates a cluster object.
 */
public void buildCluster()
{
  try {
    if (protocolVersion != null && protocolVersion.length() != 0) {
      ProtocolVersion version = getCassandraProtocolVersion();
      cluster = Cluster.builder().addContactPoint(node).withCredentials(userName, password).withProtocolVersion(version).build();
    } else {
      cluster = Cluster.builder().addContactPoint(node).withCredentials(userName, password).build();
    }
  } catch (DriverException ex) {
    throw new RuntimeException("closing database resource", ex);
  } catch (Throwable t) {
    DTThrowable.rethrow(t);
  }
}
 

@Override
public void storeAggregate()
{
  try {
    for (T tuple : tuples) {
      Mutation mutation = operationMutation(tuple);
      store.getBatchwriter().addMutation(mutation);
    }
    store.getBatchwriter().flush();

  } catch (MutationsRejectedException e) {
    logger.error("unable to write mutations", e);
    DTThrowable.rethrow(e);
  }
  tuples.clear();
}
 

@SuppressWarnings("unchecked")
private void replay(long windowId)
{
  Map<Long, byte[]> recoveredData;
  try {
    recoveredData = (Map<Long, byte[]>)this.windowDataManager.retrieve(windowId);
    if (recoveredData == null) {
      return;
    }
    for (Entry<Long, byte[]>  recoveredEntry : recoveredData.entrySet()) {
      recoveredTags.add(recoveredEntry.getKey());
      emitTuple(recoveredEntry.getValue());
    }
  } catch (IOException e) {
    DTThrowable.rethrow(e);
  }
}
 

/**
 * This executes the query to retrieve result from database.
 * It then converts each row into tuple and emit that into output port.
 */
@Override
public void emitTuples()
{
  Statement query = queryToRetrieveData();
  logger.debug(String.format("select statement: %s", query.toString()));
  RecordSet rs;
  try {
    rs = store.getClient().query(null, query);
    while (rs.next()) {
      Record rec = rs.getRecord();
      T tuple = getTuple(rec);
      outputPort.emit(tuple);
    }
  } catch (Exception ex) {
    store.disconnect();
    DTThrowable.rethrow(ex);
  }
}
 

@SuppressWarnings("ThrowFromFinallyBlock")
@Override
public void teardown()
{
  try {
    scanner.teardown();
  } catch (Throwable t) {
    DTThrowable.rethrow(t);
  } finally {
    try {
      fs.close();
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
  }
}
 

@Override
public Scanner getScanner(Connector conn)
{
  Authorizations auths = new Authorizations();
  Scanner scan = null;
  try {
    scan = conn.createScanner(getStore().getTableName(), auths);
  } catch (TableNotFoundException e) {
    logger.error("table not found ");
    DTThrowable.rethrow(e);
  }
  scan.setRange(new Range());
  // scan.fetchColumnFamily("attributes");

  return scan;
}
 

@Override
public void connect() throws IOException
{
  try {

    rengine = REngine.getLastEngine();
    if (rengine == null) {
      // new R-engine
      rengine = REngine.engineForClass(rEngineClassName, args, callBacks, runREPL);
      log.info("Creating new Rengine");
    } else {
      log.info("Got last Rengine");
    }
  } catch (Exception exc) {
    log.error("Exception: ", exc);
    DTThrowable.rethrow(exc);
  }

}
 

public static AccumuloTuple getAccumuloTuple(String row, String colFam,
    String colName)
{
  Authorizations auths = new Authorizations();

  Scanner scan = null;
  try {
    scan = con.createScanner("tab1", auths);
  } catch (TableNotFoundException e) {
    logger.error("error in test helper");
    DTThrowable.rethrow(e);
  }

  scan.setRange(new Range(new Text(row)));
  scan.fetchColumn(new Text(colFam), new Text(colName));
  // assuming only one row
  for (Entry<Key, Value> entry : scan) {
    AccumuloTuple tuple = new AccumuloTuple();
    tuple.setRow(entry.getKey().getRow().toString());
    tuple.setColFamily(entry.getKey().getColumnFamily().toString());
    tuple.setColName(entry.getKey().getColumnQualifier().toString());
    tuple.setColValue(entry.getValue().toString());
    return tuple;
  }
  return null;
}
 

private void replay(long windowId)
{
  try {
    // For first recovered window, offset is already part of recovery state.
    // So skip reading from idempotency manager
    if (!skipOffsetRecovery) {
      // Begin offset for this window is recovery offset stored for the last
      // window
      RecoveryState recoveryStateForLastWindow = (RecoveryState)getWindowDataManager().retrieve(windowId - 1);
      recoveryState.scanOffsetAtBeginWindow = recoveryStateForLastWindow.scanOffsetAtBeginWindow;
    }
    skipOffsetRecovery = false;
    RecoveryState recoveryStateForCurrentWindow = (RecoveryState)getWindowDataManager().retrieve(windowId);
    recoveryState.numberOfScanCallsInWindow = recoveryStateForCurrentWindow.numberOfScanCallsInWindow;
    if (recoveryState.scanOffsetAtBeginWindow != null) {
      scanOffset = recoveryState.scanOffsetAtBeginWindow;
    }
    replay = true;

  } catch (IOException e) {
    DTThrowable.rethrow(e);
  }
}
 

@Override
public void beginWindow(long windowId)
{
  currentWindowId = windowId;

  // if the current window is now less than the largest window, then we need to replay data
  if (currentWindowId <= windowDataManager.getLargestCompletedWindow()) {
    try {
      List<T> recoveredData =  (List<T>)this.windowDataManager.retrieve(windowId);
      if (recoveredData == null) {
        return;
      }

      // if we recovered tuples then load them to be processed by next call to emitTuples()
      recoveredTuples.addAll(recoveredData);
    } catch (IOException e) {
      DTThrowable.rethrow(e);
    }
  }
}
 

private void insertRecordsInTable()
{
  try {
    Statement stmt = dbloader.getConnection().createStatement();
    String tbName = dbloader.getTableName();

    for (int i = 0; i < id.length; i++) {
      String sql = "INSERT INTO " + tbName + " (ID,NAME,AGE,ADDRESS,SALARY) " +
          "VALUES (" + id[i] + ", '" + name[i] + "', " + age[i] + ", '" + address[i] + "', " + salary[i] + " );";
      stmt.executeUpdate(sql);
    }
  } catch (Throwable e) {
    DTThrowable.rethrow(e);
  }

}
 

private void scanKeysFromOffset()
{
  if (!scanComplete) {
    if (replay && scanCallsInCurrentWindow >= recoveryState.numberOfScanCallsInWindow) {
      try {
        Thread.sleep(sleepTimeMillis);
      } catch (InterruptedException e) {
        DTThrowable.rethrow(e);
      }
      return;
    }

    ScanResult<String> result = store.ScanKeys(scanOffset, scanParameters);
    backupOffset = scanOffset;
    scanOffset = Integer.parseInt(result.getStringCursor());
    if (scanOffset == 0) {
      // Redis store returns 0 after all data is read
      scanComplete = true;

      // point scanOffset to the end in this case for reading any new tuples
      scanOffset = backupOffset + result.getResult().size();
    }
    keys = result.getResult();
  }
  scanCallsInCurrentWindow++;
}
 

@Override
public void endWindow()
{
  // if replaying then nothing to do
  if (currentWindowId <= largestRecoveryWindowId) {
    return;
  }

  // if processing a new window then give sub-classes a chance to take action
  endNewWindow();

  // mark that we processed the window
  try {
    windowDataManager.save("processedWindow", currentWindowId);
  } catch (IOException e) {
    DTThrowable.rethrow(e);
  }
}
 

@Override
public void endWindow()
{
  while (replay && scanCallsInCurrentWindow < recoveryState.numberOfScanCallsInWindow) {
    // If less keys got scanned in this window, scan till recovery offset
    scanKeysFromOffset();
    processTuples();
  }
  super.endWindow();
  recoveryState.scanOffsetAtBeginWindow = scanOffset;
  recoveryState.numberOfScanCallsInWindow = scanCallsInCurrentWindow;

  if (currentWindowId > getWindowDataManager().getLargestCompletedWindow()) {
    try {
      getWindowDataManager().save(recoveryState, currentWindowId);
    } catch (IOException e) {
      DTThrowable.rethrow(e);
    }
  }
}
 

@BeforeClass
public static void setup()
{
  @SuppressWarnings("UnusedDeclaration")
  Class<?> clazz = org.codehaus.janino.CompilerFactory.class;
  try {
    cluster = Cluster.builder()
            .addContactPoint(NODE).build();
    session = cluster.connect(KEYSPACE);

    String createMetaTable = "CREATE TABLE IF NOT EXISTS " + CassandraTransactionalStore.DEFAULT_META_TABLE + " ( "
        + CassandraTransactionalStore.DEFAULT_APP_ID_COL + " TEXT, "
        + CassandraTransactionalStore.DEFAULT_OPERATOR_ID_COL + " INT, "
        + CassandraTransactionalStore.DEFAULT_WINDOW_COL + " BIGINT, "
        + "PRIMARY KEY (" + CassandraTransactionalStore.DEFAULT_APP_ID_COL + ", " + CassandraTransactionalStore.DEFAULT_OPERATOR_ID_COL + ") "
        + ");";
    session.execute(createMetaTable);
    String createTable = "CREATE TABLE IF NOT EXISTS " + KEYSPACE + "." + TABLE_NAME + " (id uuid PRIMARY KEY,age int,lastname text,test boolean,floatvalue float,doubleValue double,set1 set<int>,list1 list<int>,map1 map<text,int>,last_visited timestamp);";
    session.execute(createTable);
    createTable = "CREATE TABLE IF NOT EXISTS " + KEYSPACE + "." + TABLE_NAME_INPUT + " (id int PRIMARY KEY,lastname text,age int);";
    session.execute(createTable);
  } catch (Throwable e) {
    DTThrowable.rethrow(e);
  }
}
 

@BeforeClass
public static void setup()
{
  try {
    Class.forName(DB_DRIVER).newInstance();

    Connection con = DriverManager.getConnection(DB_URL);
    Statement stmt = con.createStatement();

    String createMetaTable = "CREATE TABLE IF NOT EXISTS " + JdbcTransactionalStore.DEFAULT_META_TABLE + " ( "
        + JdbcTransactionalStore.DEFAULT_APP_ID_COL + " VARCHAR(100) NOT NULL, "
        + JdbcTransactionalStore.DEFAULT_OPERATOR_ID_COL + " INT NOT NULL, "
        + JdbcTransactionalStore.DEFAULT_WINDOW_COL + " BIGINT NOT NULL, "
        + "UNIQUE (" + JdbcTransactionalStore.DEFAULT_APP_ID_COL + ", "
        + JdbcTransactionalStore.DEFAULT_OPERATOR_ID_COL + ", " + JdbcTransactionalStore.DEFAULT_WINDOW_COL + ") "
        + ")";
    stmt.executeUpdate(createMetaTable);

    String createTable = "CREATE TABLE IF NOT EXISTS " + TABLE_NAME
        + " (ACCOUNT_NO INTEGER, NAME VARCHAR(255),AMOUNT INTEGER)";
    stmt.executeUpdate(createTable);

  } catch (Throwable e) {
    DTThrowable.rethrow(e);
  }
}
 

@Override
protected void finished(Description description)
{
  try {
    // Clean up
    session.close();
    connection.close();
  } catch (JMSException ex) {
    DTThrowable.rethrow(ex);
  }
  operator.deactivate();
  operator.teardown();
  try {
    FileUtils.deleteDirectory(new File("target/" + description.getClassName()));
    testBase.afterTest();
  } catch (Exception e) {
    DTThrowable.rethrow(e);
  }
}
 

@Override
public byte[] getTuple(byte[] message)
{
  if (registry == null) {
    return message;
  }
  String inputString = new String(message);
  try {
    JSONObject jSONObject = new JSONObject(inputString);
    int typeId = registry.getIndex(LogstreamUtil.LOG_TYPE, routingKey);
    jSONObject.put(LogstreamUtil.LOG_TYPE, typeId);
    String outputString = jSONObject.toString();
    message = outputString.getBytes();
  }
  catch (Throwable ex) {
    DTThrowable.rethrow(ex);
  }

  return message;
}
 

@Override
public void processTuples()
{
  for (String key : keys) {
    if (store.getType(key).equals("hash")) {
      Map<String, String> mapValue = store.getMap(key);
      if (isFirstTuple) {
        try {
          processFirstTuple(mapValue);
        } catch (ClassNotFoundException e) {
          DTThrowable.rethrow(e);
        }
      }
      isFirstTuple = false;
      outputPort.emit(new KeyValPair<String, Object>(key, convertMapToObject(mapValue)));
    }
  }
  keys.clear();
}
 

/**
 * returns the number value of the supplied object
 * @param value
 * @return
 */
public static Number extractNumber(Object value)
{
  NumberFormat numberFormat = NumberFormat.getInstance();
  if (value instanceof Number) {
    return (Number)value;
  }
  else if (value == null) {
    return new Long(0);
  }
  else {
    try {
      return numberFormat.parse(value.toString());
    }
    catch (ParseException ex) {
      DTThrowable.rethrow(ex);
    }
  }
  return new Long(0);
}
 

@BeforeClass
public static void setup() {
  try {
    Class.forName(DB_DRIVER).newInstance();

    Connection con = DriverManager.getConnection(DB_URL);
    Statement stmt = con.createStatement();

    String createMetaTable = "CREATE TABLE IF NOT EXISTS " + JdbcTransactionalStore.DEFAULT_META_TABLE + " ( "
            + JdbcTransactionalStore.DEFAULT_APP_ID_COL + " VARCHAR(100) NOT NULL, "
            + JdbcTransactionalStore.DEFAULT_OPERATOR_ID_COL + " INT NOT NULL, "
            + JdbcTransactionalStore.DEFAULT_WINDOW_COL + " BIGINT NOT NULL, "
            + "UNIQUE (" + JdbcTransactionalStore.DEFAULT_APP_ID_COL + ", "
            + JdbcTransactionalStore.DEFAULT_OPERATOR_ID_COL + ", " + JdbcTransactionalStore.DEFAULT_WINDOW_COL + ") "
            + ")";
    stmt.executeUpdate(createMetaTable);

    String createTable = "CREATE TABLE IF NOT EXISTS " + TABLE_NAME
            + " (ACCOUNT_NO INTEGER, NAME VARCHAR(255),AMOUNT INTEGER)";
    stmt.executeUpdate(createTable);

  } catch (Throwable e) {
    DTThrowable.rethrow(e);
  }
}
 

@Override
public void connect() throws IOException
{
  super.connect();
  logger.debug("connection established");
  try {
    CouchbaseConnectionFactoryBuilder cfb = new CouchbaseConnectionFactoryBuilder();
    cfb.setOpTimeout(timeout);  // wait up to 10 seconds for an operation to succeed
    cfb.setOpQueueMaxBlockTime(blockTime); // wait up to 10 second when trying to enqueue an operation
    clientMeta = new CouchbaseClient(cfb.buildCouchbaseConnection(baseURIs, bucketMeta, passwordMeta));
  } catch (IOException e) {
    logger.error("Error connecting to Couchbase: ", e);
    DTThrowable.rethrow(e);
  }
}
 

/**
 * This executes the query to retrieve result from database.
 * It then converts each row into tuple and emit that into output port.
 */
@Override
public void emitTuples()
{
  String query = queryToRetrieveData();
  logger.debug("select statement: {}", query);

  SimpleStatement stmt = new SimpleStatement(query);
  stmt.setFetchSize(fetchSize);
  try {
    if (nextPageState != null) {
      stmt.setPagingState(nextPageState);
    }
    ResultSet result = store.getSession().execute(stmt);
    nextPageState = result.getExecutionInfo().getPagingState();

    if (!result.isExhausted()) {
      for (Row row : result) {
        T tuple = getTuple(row);
        emit(tuple);
      }
    } else {
      // No rows available wait for some time before retrying so as to not continuously slam the database
      Thread.sleep(waitForDataTimeout);
    }
  } catch (Exception ex) {
    store.disconnect();
    DTThrowable.rethrow(ex);
  }
}
 

@Override
public void setup(OperatorContext context)
{
  JAXBContext ctx;
  try {
    ctx = JAXBContext.newInstance(getClazz());
    marshaller = ctx.createMarshaller();
    marshaller.setProperty(Marshaller.JAXB_FRAGMENT, true);
    marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, prettyPrint);
  } catch (JAXBException e) {
    DTThrowable.wrapIfChecked(e);
  }
}