org.apache.apex.malhar.lib.dimensions.DimensionsEvent.Aggregate Java Examples

@Override
public Aggregate getGroup(InputEvent src, int aggregatorIndex)
{
  src.used = true;
  int[] stringIndexSubset = this.context.indexSubsetKeys.fieldsStringIndexSubset;

  String[] keys;

  if (stringIndexSubset == null) {
    keys = new String[0];
  } else {
    keys = new String[stringIndexSubset.length];
  }

  for (int counter = 0; counter < keys.length; counter++) {
    keys[counter] = src.getKeys().getFieldsString()[stringIndexSubset[counter]];
  }

  MachineAggregate machineAggregate = new MachineAggregate(keys, 0, context.schemaID, context.dimensionsDescriptorID,
      context.aggregatorID, 0L, 0L, 0L,
      this.context.dd.getCustomTimeBucket().roundDown(src.getKeys().getFieldsLong()[0]),
      this.context.customTimeBucketRegistry.getTimeBucketId(this.context.dd.getCustomTimeBucket()));

  machineAggregate.setAggregatorIndex(aggregatorIndex);
  return machineAggregate;
}
 

/**
 * update existed sub aggregate.
 * The sub aggregates which kept in composite aggregate as candidate could be changed. synchronize the value with
 * input aggregates.
 *
 * @param resultAggregate
 * @param valueField
 * @param inputSubEventKeys
 * @param inputAggregatesRepo
 */
@SuppressWarnings("unchecked")
protected void updateAggregate(Aggregate resultAggregate, String valueField,
    Set<EventKey> inputSubEventKeys, Map<EventKey, Aggregate> inputAggregatesRepo)
{
  Map<String, Object> resultAggregateFieldToValue =
      (Map<String, Object>)resultAggregate.getAggregates().getFieldObject(valueField);
  if (resultAggregateFieldToValue == null) {
    return;
  }

  for (EventKey inputSubEventKey : inputSubEventKeys) {
    Aggregate inputSubAggregate = inputAggregatesRepo.get(inputSubEventKey);
    String mapKey = getStoreMapKey(inputSubAggregate.getEventKey(),
        resultAggregate.getEventKey().getKey().getFieldDescriptor().getFieldList());
    //Aggregate existedAggregate = existedSubEventKeyToAggregate.get(inputSubEventKey);
    if (resultAggregateFieldToValue.get(mapKey) != null) {
      resultAggregateFieldToValue.put(mapKey, inputSubAggregate.getAggregates().getField(valueField));
    }
  }
}
 

/**
 * The result keep a list of object for each aggregate value
 * The value of resultAggregate should keep a list of inputEventKey(the value can be get from cache or load) or a map
 * from inputEventKey to the value instead of just a list of aggregate value. As the value could be changed in
 * current window, and this change should be applied.
 *
 * precondition: resultAggregate.eventKey matches with inputSubEventKeys
 * notes: this algorithm only support TOP for positive values and BOTTOM for negative values
 */
@Override
public void aggregate(Aggregate resultAggregate, Set<EventKey> inputSubEventKeys,
    Map<EventKey, Aggregate> inputAggregatesRepo)
{
  //there are problem for composite's value field descriptor, just ignore now.
  GPOMutable resultGpo = resultAggregate.getAggregates();
  final List<String> compositeFieldList = resultAggregate.getEventKey().getKey().getFieldDescriptor().getFieldList();

  //Map<EventKey, Aggregate> existedSubEventKeyToAggregate = Maps.newHashMap();
  for (String valueField : resultGpo.getFieldDescriptor().getFieldList()) {
    //the resultGpo keep a list of sub aggregates
    updateAggregate(resultAggregate, valueField, inputSubEventKeys, inputAggregatesRepo);

    //compare the existed sub aggregates with the new input aggregates to update the list
    for (EventKey eventKey : inputSubEventKeys) {
      aggregate(compositeFieldList, resultGpo, eventKey, inputAggregatesRepo.get(eventKey).getAggregates());
    }
  }

}
 

/**
 * Creates an {@link Aggregate} from the given {@link InputEvent}.
 *
 * @param inputEvent      The {@link InputEvent} to unpack into an {@link Aggregate}.
 * @param context         The conversion context required to transform the {@link InputEvent} into
 *                        the correct {@link Aggregate}.
 * @param aggregatorIndex The aggregatorIndex assigned to this {@link Aggregate}.
 * @return The converted {@link Aggregate}.
 */
public static Aggregate createAggregate(InputEvent inputEvent,
    DimensionsConversionContext context,
    int aggregatorIndex)
{
  GPOMutable aggregates = new GPOMutable(context.aggregateDescriptor);
  EventKey eventKey = createEventKey(inputEvent,
      context,
      aggregatorIndex);

  Aggregate aggregate = new Aggregate(eventKey,
      aggregates);
  aggregate.setAggregatorIndex(aggregatorIndex);

  return aggregate;
}
 

@Override
public Aggregate getGroup(InputEvent src, int aggregatorIndex)
{
  src.used = true;
  GPOMutable aggregates = new GPOMutable(context.aggregateDescriptor);
  GPOMutable keys = new GPOMutable(context.keyDescriptor);
  GPOUtils.indirectCopy(keys, src.getKeys(), context.indexSubsetKeys);

  EventKey eventKey = createEventKey(src,
      context,
      aggregatorIndex);

  long[] longFields = aggregates.getFieldsLong();

  for (int index = 0;
      index < longFields.length;
      index++) {
    longFields[index] = 0;
  }

  return new Aggregate(eventKey,
      aggregates);
}
 

@Override
public Slice toByteArray(Aggregate o)
{
  final Output output = new Output(32, -1);
  try {
    kryo.writeClassAndObject(output, o);
  } finally {
    output.close();
  }
  return new Slice(output.getBuffer(), 0, output.position());
}
 

@Override
public void aggregate(Aggregate dest, InputEvent src)
{
  ((MachineAggregate)dest).cpuUsage += src.getAggregates().getFieldsLong()[0];
  ((MachineAggregate)dest).hddUsage += src.getAggregates().getFieldsLong()[1];
  ((MachineAggregate)dest).ramUsage += src.getAggregates().getFieldsLong()[2];
}
 

@Override
public void aggregate(Aggregate dest, Aggregate src)
{
  ((MachineAggregate)dest).cpuUsage += ((MachineAggregate)src).cpuUsage;
  ((MachineAggregate)dest).hddUsage += ((MachineAggregate)src).hddUsage;
  ((MachineAggregate)dest).ramUsage += ((MachineAggregate)src).ramUsage;
}
 

@Override
public Aggregate getGroup(InputEvent src, int aggregatorIndex)
{
  src.used = true;
  Aggregate aggregate = createAggregate(src,
      context,
      aggregatorIndex);

  GPOMutable value = aggregate.getAggregates();
  GPOUtils.zeroFillNumeric(value);

  return aggregate;
}
 

@Override
public Aggregate getGroup(InputEvent src, int aggregatorIndex)
{
  Aggregate aggregate = super.getGroup(src, aggregatorIndex);

  GPOUtils.indirectCopy(aggregate.getAggregates(), src.getAggregates(), context.indexSubsetAggregates);

  return aggregate;
}
 

@Override
public void endWindow()
{
  long endWindowStartTime = System.currentTimeMillis();
  
  super.endWindow();
  
  //update bucket update time;
  for(Aggregate aggregate : aggregateList) {
    updateUpdateTime(getKey(aggregate), getBucket(aggregate), System.currentTimeMillis());
  }
  
  aggregateList.clear();
  if(++windowCountForLog == logUpdateWindows){
    //logUpdateTime();
    logger.info("Logging Times: {}", loggedTimes);
    logFinalLatencies();
    logger.info("Logging Times: {}", loggedTimes);
    windowCountForLog = 0;
    ++loggedTimes;
  }
 
  //window set to 200
  long windowPeriod = System.currentTimeMillis() - lastEndWindowTime;
  if(windowPeriod  > 400) {
    logger.warn("Unexpected long window period: {}, endWindow executed time: {}", windowPeriod, System.currentTimeMillis() - endWindowStartTime);
  }
  lastEndWindowTime = System.currentTimeMillis();
}
 

@Override
public Aggregate getGroup(InputEvent src, int aggregatorIndex)
{
  Aggregate aggregate = super.getGroup(src, aggregatorIndex);

  GPOUtils.indirectCopy(aggregate.getAggregates(), src.getAggregates(), context.indexSubsetAggregates);

  return aggregate;
}
 

@Override
public Aggregate getGroup(InputEvent src, int aggregatorIndex)
{
  src.used = true;
  int[] stringIndexSubset = this.context.indexSubsetKeys.fieldsStringIndexSubset;

  String[] keys;

  if (stringIndexSubset == null) {
    keys = new String[0];
  } else {
    keys = new String[stringIndexSubset.length];
  }

  for (int counter = 0; counter < keys.length; counter++) {
    keys[counter] = src.getKeys().getFieldsString()[stringIndexSubset[counter]];
  }

  MachineAggregate machineAggregate = new MachineAggregate(keys, 0, context.schemaID, context.dimensionsDescriptorID,
      context.aggregatorID, src.getAggregates().getFieldsLong()[0], src.getAggregates().getFieldsLong()[2],
      src.getAggregates().getFieldsLong()[1],
      this.context.dd.getCustomTimeBucket().roundDown(src.getEventKey().getKey().getFieldsLong()[0]),
      this.context.customTimeBucketRegistry.getTimeBucketId(this.context.dd.getCustomTimeBucket()));

  machineAggregate.setAggregatorIndex(aggregatorIndex);
  return machineAggregate;
}
 

@Override
public Aggregate getGroup(InputEvent src, int aggregatorIndex)
{
  src.used = true;
  Aggregate aggregate = createAggregate(src,
      context,
      aggregatorIndex);
  return aggregate;
}
 

@Override
public Aggregate getGroup(InputEvent src, int aggregatorIndex)
{
  src.used = true;
  Aggregate agg = createAggregate(src,
      context,
      aggregatorIndex);

  GPOUtils.indirectCopy(agg.getAggregates(), src.getAggregates(), context.indexSubsetAggregates);

  GPOMutable metaData = new GPOMutable(getMetaDataDescriptor());

  GPOMutable fullKey = new GPOMutable(src.getKeys());

  if (context.inputTimestampIndex >= 0) {
    fullKey.getFieldsLong()[context.inputTimestampIndex] = -1L;
  }

  List<GPOMutable> keys = Lists.newArrayList(fullKey);

  GPOMutable value = new GPOMutable(agg.getAggregates());
  List<GPOMutable> values = Lists.newArrayList(value);

  metaData.getFieldsObject()[KEY_FD_INDEX] = fullKey.getFieldDescriptor();
  metaData.getFieldsObject()[AGGREGATE_FD_INDEX] = value.getFieldDescriptor();
  metaData.getFieldsObject()[KEYS_INDEX] = keys;
  metaData.getFieldsObject()[AGGREGATES_INDEX] = values;
  agg.setMetaData(metaData);

  return agg;
}
 

@Override
public void aggregate(Aggregate dest, InputEvent src)
{
  @SuppressWarnings("unchecked")
  List<GPOMutable> destKeys =
      (List<GPOMutable>)dest.getMetaData().getFieldsObject()[KEYS_INDEX];

  @SuppressWarnings("unchecked")
  List<GPOMutable> destAggregates =
      (List<GPOMutable>)dest.getMetaData().getFieldsObject()[AGGREGATES_INDEX];

  long timestamp = 0L;

  if (context.inputTimestampIndex >= 0) {
    timestamp = src.getKeys().getFieldsLong()[context.inputTimestampIndex];
    src.getKeys().getFieldsLong()[context.inputTimestampIndex] = -1L;
  }

  if (!contains(destKeys, src.getKeys())) {
    destKeys.add(new GPOMutable(src.getKeys()));

    GPOMutable aggregates = new GPOMutable(context.aggregateDescriptor);
    GPOUtils.indirectCopy(aggregates, src.getAggregates(), context.indexSubsetAggregates);

    destAggregates.add(aggregates);

    this.aggregateAggs(dest.getAggregates(), aggregates);
  }

  if (context.inputTimestampIndex >= 0) {
    src.getKeys().getFieldsLong()[context.inputTimestampIndex] = timestamp;
  }
}
 

@Override
public Aggregate getGroup(InputEvent src, int aggregatorIndex)
{
  Aggregate aggregate = super.getGroup(src, aggregatorIndex);

  GPOUtils.indirectCopy(aggregate.getAggregates(), src.getAggregates(), context.indexSubsetAggregates);

  return aggregate;
}
 

/**
 * Sets the parameters on the {@link java.sql.PreparedStatement} based on the
 * values in the given {@link Aggregate}.
 *
 * @param aggregate
 *          The {@link Aggregate} whose values will be set on the
 *          corresponding {@link java.sql.PreparedStatement}.
 */
private void setStatementParameters(Aggregate aggregate)
{
  EventKey eventKey = aggregate.getEventKey();

  int ddID = eventKey.getDimensionDescriptorID();
  int aggID = eventKey.getAggregatorID();

  LOG.info("Setting statement params {} {}", ddID, aggID);

  FieldsDescriptor keyFD = schema.getDimensionsDescriptorIDToKeyDescriptor().get(ddID);
  FieldsDescriptor aggFD = schema.getDimensionsDescriptorIDToAggregatorIDToOutputAggregatorDescriptor().get(ddID)
      .get(aggID);

  GPOMutable key = eventKey.getKey();
  key.setFieldDescriptor(keyFD);

  GPOMutable value = aggregate.getAggregates();
  value.setFieldDescriptor(aggFD);

  int qCounter = 1;

  PreparedStatement ps = ddIDToAggIDToStatement.get(ddID).get(aggID);

  try {
    qCounter = setParams(ps, key, qCounter, true);
    setParams(ps, value, qCounter, false);
    ps.addBatch();
  } catch (SQLException ex) {
    throw new RuntimeException(ex);
  }
}
 

@Override
public void aggregate(Aggregate dest, Aggregate src)
{
  ((MachineAggregate)dest).cpuUsage += ((MachineAggregate)src).cpuUsage;
  ((MachineAggregate)dest).hddUsage += ((MachineAggregate)src).hddUsage;
  ((MachineAggregate)dest).ramUsage += ((MachineAggregate)src).ramUsage;
}
 

@Override
public void aggregate(Aggregate dest, InputEvent src)
{
  ((MachineAggregate)dest).cpuUsage++;
  ((MachineAggregate)dest).hddUsage++;
  ((MachineAggregate)dest).ramUsage++;
}
 

@Override
public void aggregate(Aggregate dest, InputEvent src)
{
  GPOMutable destAggs = dest.getAggregates();
  GPOMutable srcAggs = src.getAggregates();

  aggregateInput(destAggs, srcAggs);
}
 

@Override
public void process(AdInfoAggregateEvent tuple)
{
  int ddID = prevDdIDToThisDdID.get(tuple.getDimensionsDescriptorID());
  FieldsDescriptor keyDescriptor = dimensionsConfigurationSchema.getDimensionsDescriptorIDToKeyDescriptor().get(ddID);

  GPOMutable key = new GPOMutable(keyDescriptor);

  for (String field : keyDescriptor.getFieldList()) {
    if (field.equals(InputItemGenerator.PUBLISHER)) {
      key.setField(InputItemGenerator.PUBLISHER, tuple.publisher);
    } else if (field.equals(InputItemGenerator.ADVERTISER)) {
      key.setField(InputItemGenerator.ADVERTISER, tuple.advertiser);
    } else if (field.equals(InputItemGenerator.LOCATION)) {
      key.setField(InputItemGenerator.LOCATION, tuple.location);
    }
  }

  key.setField(DimensionsDescriptor.DIMENSION_TIME, tuple.time);
  key.setField(DimensionsDescriptor.DIMENSION_TIME_BUCKET, tuple.timeBucket);

  EventKey eventKey = new EventKey(schemaID, ddID, sumAggregatorIndex, key);

  GPOMutable aggregates = new GPOMutable(aggregateFieldsDescriptor);
  aggregates.setField(InputItemGenerator.IMPRESSIONS, tuple.impressions);
  aggregates.setField(InputItemGenerator.COST, tuple.cost);
  aggregates.setField(InputItemGenerator.REVENUE, tuple.revenue);
  aggregates.setField(InputItemGenerator.CLICKS, tuple.clicks);

  outputPort.emit(new Aggregate(eventKey, aggregates));
}
 

@Override
public void populateDAG(DAG dag, Configuration conf)
{
  InputItemGenerator input = dag.addOperator("InputGenerator", InputItemGenerator.class);
  DimensionsComputationFlexibleSingleSchemaPOJO dimensions = dag.addOperator("DimensionsComputation",
      DimensionsComputationFlexibleSingleSchemaPOJO.class);
  dag.getMeta(dimensions).getAttributes().put(Context.OperatorContext.APPLICATION_WINDOW_COUNT, 10);
  DevNull<Object> devNull = dag.addOperator("DevNull", new DevNull<Object>());

  //Set input properties
  String eventSchema = SchemaUtils.jarResourceFileToString("adsBenchmarkSchema.json");
  input.setEventSchemaJSON(eventSchema);

  Map<String, String> keyToExpression = Maps.newHashMap();
  keyToExpression.put("publisher", "getPublisher()");
  keyToExpression.put("advertiser", "getAdvertiser()");
  keyToExpression.put("location", "getLocation()");
  keyToExpression.put("time", "getTime()");

  Map<String, String> aggregateToExpression = Maps.newHashMap();
  aggregateToExpression.put("cost", "getCost()");
  aggregateToExpression.put("revenue", "getRevenue()");
  aggregateToExpression.put("impressions", "getImpressions()");
  aggregateToExpression.put("clicks", "getClicks()");

  DimensionsComputationUnifierImpl<InputEvent, Aggregate> unifier = new DimensionsComputationUnifierImpl<InputEvent, Aggregate>();
  dimensions.setUnifier(unifier);
  dimensions.setKeyToExpression(keyToExpression);
  dimensions.setAggregateToExpression(aggregateToExpression);
  dimensions.setConfigurationSchemaJSON(eventSchema);

  dag.addStream("InputStream", input.outputPort, dimensions.input).setLocality(Locality.CONTAINER_LOCAL);
  dag.addStream("DimensionalData", dimensions.output, devNull.data);
}
 

@Override
public void processTuple(Aggregate tuple)
{
  tuples.add(tuple);
  if ((tuples.size() - batchStartIdx) >= batchSize) {
    processBatch();
  }
}
 

@Override
public void aggregate(Aggregate dest, Aggregate src)
{
  GPOMutable destAggs = dest.getAggregates();
  GPOMutable srcAggs = src.getAggregates();

  aggregateAggs(destAggs, srcAggs);
}
 

@Override
public void aggregate(Aggregate dest, InputEvent src)
{
  long[] fieldsLong = dest.getAggregates().getFieldsLong();

  for (int index = 0;
      index < fieldsLong.length;
      index++) {
    //increment count
    fieldsLong[index]++;
  }
}
 

@Override
protected void emitUpdates()
{
  super.emitUpdates();

  for (int index = 0; index < aggregatorsInfo.size(); ++index) {
    MutablePair<Integer, Integer> info = aggregatorsInfo.get(index);
    if (info == null) {
      continue;
    }

    int aggregatorID = info.left;
    int dimensionDescriptorID = info.right;
    DefaultOutputPort<List<Aggregate>> outputPort = getOutputPort(index++, aggregatorID, dimensionDescriptorID);
    if (outputPort == null || !outputPort.isConnected()) {
      continue;
    }

    updatingAggregates.clear();

    for (Map.Entry<EventKey, Aggregate> entry : cache.entrySet()) {
      if (aggregatorID == entry.getKey().getAggregatorID()
          && entry.getKey().getDimensionDescriptorID() == dimensionDescriptorID
          && getMaxTimestamp() == entry.getKey().getKey().getFieldLong(TIME_FIELD_NAME)) {
        updatingAggregates.add(entry.getValue());
      }
    }

    if (!updatingAggregates.isEmpty()) {
      outputPort.emit(updatingAggregates);
    }
  }
}
 

@Override
public void aggregate(Aggregate destAgg, Aggregate srcAgg)
{
  long[] destLongs = destAgg.getAggregates().getFieldsLong();
  long[] srcLongs = srcAgg.getAggregates().getFieldsLong();

  for (int index = 0;
      index < destLongs.length;
      index++) {
    //aggregate count
    destLongs[index] += srcLongs[index];
  }
}
 

@Override
public Aggregate getGroup(InputEvent src, int aggregatorIndex)
{
  Aggregate aggregate = super.getGroup(src, aggregatorIndex);

  GPOUtils.indirectCopy(aggregate.getAggregates(), src.getAggregates(), context.indexSubsetAggregates);

  return aggregate;
}
 

@Override
public void aggregate(Aggregate dest, InputEvent src)
{
  //Ignore
}