Java Code Examples for org.apache.apex.malhar.lib.dimensions.DimensionsEvent.Aggregate#getAggregates()

/**
 * The result keep a list of object for each aggregate value
 * The value of resultAggregate should keep a list of inputEventKey(the value can be get from cache or load) or a map
 * from inputEventKey to the value instead of just a list of aggregate value. As the value could be changed in
 * current window, and this change should be applied.
 *
 * precondition: resultAggregate.eventKey matches with inputSubEventKeys
 * notes: this algorithm only support TOP for positive values and BOTTOM for negative values
 */
@Override
public void aggregate(Aggregate resultAggregate, Set<EventKey> inputSubEventKeys,
    Map<EventKey, Aggregate> inputAggregatesRepo)
{
  //there are problem for composite's value field descriptor, just ignore now.
  GPOMutable resultGpo = resultAggregate.getAggregates();
  final List<String> compositeFieldList = resultAggregate.getEventKey().getKey().getFieldDescriptor().getFieldList();

  //Map<EventKey, Aggregate> existedSubEventKeyToAggregate = Maps.newHashMap();
  for (String valueField : resultGpo.getFieldDescriptor().getFieldList()) {
    //the resultGpo keep a list of sub aggregates
    updateAggregate(resultAggregate, valueField, inputSubEventKeys, inputAggregatesRepo);

    //compare the existed sub aggregates with the new input aggregates to update the list
    for (EventKey eventKey : inputSubEventKeys) {
      aggregate(compositeFieldList, resultGpo, eventKey, inputAggregatesRepo.get(eventKey).getAggregates());
    }
  }

}
 

@Override
public Aggregate getGroup(InputEvent src, int aggregatorIndex)
{
  src.used = true;
  Aggregate agg = createAggregate(src,
      context,
      aggregatorIndex);

  GPOUtils.indirectCopy(agg.getAggregates(), src.getAggregates(), context.indexSubsetAggregates);

  GPOMutable metaData = new GPOMutable(getMetaDataDescriptor());

  GPOMutable fullKey = new GPOMutable(src.getKeys());

  if (context.inputTimestampIndex >= 0) {
    fullKey.getFieldsLong()[context.inputTimestampIndex] = -1L;
  }

  List<GPOMutable> keys = Lists.newArrayList(fullKey);

  GPOMutable value = new GPOMutable(agg.getAggregates());
  List<GPOMutable> values = Lists.newArrayList(value);

  metaData.getFieldsObject()[KEY_FD_INDEX] = fullKey.getFieldDescriptor();
  metaData.getFieldsObject()[AGGREGATE_FD_INDEX] = value.getFieldDescriptor();
  metaData.getFieldsObject()[KEYS_INDEX] = keys;
  metaData.getFieldsObject()[AGGREGATES_INDEX] = values;
  agg.setMetaData(metaData);

  return agg;
}
 

@Override
public Aggregate getGroup(InputEvent src, int aggregatorIndex)
{
  src.used = true;
  Aggregate aggregate = createAggregate(src,
      context,
      aggregatorIndex);

  GPOMutable value = aggregate.getAggregates();
  GPOUtils.zeroFillNumeric(value);

  return aggregate;
}
 

@Override
public void aggregate(Aggregate dest, Aggregate src)
{
  GPOMutable destAggs = dest.getAggregates();
  GPOMutable srcAggs = src.getAggregates();

  aggregateAggs(destAggs, srcAggs);
}
 

@Override
public void aggregate(Aggregate dest, InputEvent src)
{
  GPOMutable destAggs = dest.getAggregates();
  GPOMutable srcAggs = src.getAggregates();

  aggregateInput(destAggs, srcAggs);
}
 

/**
 * Sets the parameters on the {@link java.sql.PreparedStatement} based on the
 * values in the given {@link Aggregate}.
 *
 * @param aggregate
 *          The {@link Aggregate} whose values will be set on the
 *          corresponding {@link java.sql.PreparedStatement}.
 */
private void setStatementParameters(Aggregate aggregate)
{
  EventKey eventKey = aggregate.getEventKey();

  int ddID = eventKey.getDimensionDescriptorID();
  int aggID = eventKey.getAggregatorID();

  LOG.info("Setting statement params {} {}", ddID, aggID);

  FieldsDescriptor keyFD = schema.getDimensionsDescriptorIDToKeyDescriptor().get(ddID);
  FieldsDescriptor aggFD = schema.getDimensionsDescriptorIDToAggregatorIDToOutputAggregatorDescriptor().get(ddID)
      .get(aggID);

  GPOMutable key = eventKey.getKey();
  key.setFieldDescriptor(keyFD);

  GPOMutable value = aggregate.getAggregates();
  value.setFieldDescriptor(aggFD);

  int qCounter = 1;

  PreparedStatement ps = ddIDToAggIDToStatement.get(ddID).get(aggID);

  try {
    qCounter = setParams(ps, key, qCounter, true);
    setParams(ps, value, qCounter, false);
    ps.addBatch();
  } catch (SQLException ex) {
    throw new RuntimeException(ex);
  }
}
 

@Override
public GPOMutable convert(Aggregate aggregate)
{
  final FieldsDescriptor aggregatesFd = dimensitionSchema.getDimensionsDescriptorIDToAggregatorIDToOutputAggregatorDescriptor().get(aggregate.getDimensionDescriptorID()).get(aggregate.getAggregatorID());
  aggregate.getAggregates().setFieldDescriptor(aggregatesFd);

  final FieldsDescriptor keysFd = dimensitionSchema.getDimensionsDescriptorIDToKeyDescriptor().get(aggregate.getDimensionDescriptorID());
  GPOMutable keys = aggregate.getKeys();
  keys.setFieldDescriptor(keysFd);

  /****
   * { Type fieldType = keysFd.getType("deviceModel"); logger.info(
   * "The type of field '{}' is '{}'", "deviceModel", fieldType);
   * 
   * String deviceModel = keys.getFieldString("deviceModel"); Map<String,
   * Type> fieldToType = Maps.newHashMap(); fieldToType.put("deviceModel",
   * Type.STRING); fieldToType.put("downloadBytes", Type.LONG);
   * 
   * FieldsDescriptor fd = new FieldsDescriptor(fieldToType); GPOMutable gpo =
   * new GPOMutable(fd);
   * 
   * gpo.setField("deviceModel", deviceModel); gpo.setField("downloadBytes",
   * .getFieldLong("downloadBytes"));
   * 
   * return gpo; }
   */

  GPOMutable gpo;
  if (staticFields != null) {
    gpo = new GPOMutable(staticFields);
  } else {
    gpo = new GPOMutable(getFieldsDescriptor());
  }

  for (Map.Entry<MutablePair<String, Type>, MutablePair<String, Type>> entry : keyValueMap.entrySet()) {
    for (int i = 0; i < 2; ++i) {
      String fieldName;
      Type type;
      GPOMutable fieldValueSource;
      if (i == 0) {
        fieldName = entry.getKey().getKey();
        type = entry.getKey().getValue();
        fieldValueSource = keys;
      } else {
        fieldName = entry.getValue().getKey();
        type = entry.getValue().getValue();
        fieldValueSource = aggregate.getAggregates();
      }

      switch (type) {
        case BOOLEAN:
          gpo.setField(fieldName, fieldValueSource.getFieldBool(fieldName));
          break;
        case STRING:
          gpo.setField(fieldName, fieldValueSource.getFieldString(fieldName));
          break;
        case CHAR:
          gpo.setField(fieldName, fieldValueSource.getFieldChar(fieldName));
          break;
        case DOUBLE:
          gpo.setField(fieldName, fieldValueSource.getFieldDouble(fieldName));
          break;
        case FLOAT:
          gpo.setField(fieldName, fieldValueSource.getFieldFloat(fieldName));
          break;
        case LONG:
          gpo.setField(fieldName, fieldValueSource.getFieldLong(fieldName));
          break;
        case INTEGER:
          gpo.setField(fieldName, fieldValueSource.getFieldInt(fieldName));
          break;
        case SHORT:
          gpo.setField(fieldName, fieldValueSource.getFieldShort(fieldName));
          break;
        case BYTE:
          gpo.setField(fieldName, fieldValueSource.getFieldShort(fieldName));
          break;
        case OBJECT:
          gpo.setFieldObject(fieldName, fieldValueSource.getFieldObject(fieldName));
          break;
        default:
          throw new RuntimeException("Unhandled type: " + type);
      }
    }
  }
  return gpo;

}
 

@Test
public void simpleTest()
{
  AdsConverter adsConverter = new AdsConverter();

  String eventSchema = SchemaUtils.jarResourceFileToString(EVENT_SCHEMA);

  String[] dimensionSpecs = new String[] {
      "time=" + TimeUnit.MINUTES, "time=" + TimeUnit.MINUTES + ":location",
      "time=" + TimeUnit.MINUTES + ":advertiser", "time=" + TimeUnit.MINUTES + ":publisher",
      "time=" + TimeUnit.MINUTES + ":advertiser:location", "time=" + TimeUnit.MINUTES + ":publisher:location",
      "time=" + TimeUnit.MINUTES + ":publisher:advertiser",
      "time=" + TimeUnit.MINUTES + ":publisher:advertiser:location" 
      };

  AdInfoAggregateEvent aae = new AdInfoAggregateEvent();
  aae.time = 60;
  aae.timeBucket = TimeBucket.MINUTE.ordinal();
  aae.publisher = "google";
  aae.advertiser = "safeway";
  aae.location = "SKY";
  aae.impressions = 1L;
  aae.clicks = 1L;
  aae.cost = 1.0;
  aae.revenue = 1.0;
  aae.setDimensionsDescriptorID(dimensionSpecs.length - 1);

  adsConverter.setEventSchemaJSON(eventSchema);
  adsConverter.setDimensionSpecs(dimensionSpecs);

  CollectorTestSink<Aggregate> sink = new CollectorTestSink<Aggregate>();
  TestUtils.setSink(adsConverter.outputPort, sink);

  adsConverter.setup(null);
  adsConverter.beginWindow(0L);
  adsConverter.inputPort.put(aae);
  adsConverter.endWindow();

  Assert.assertEquals(1, sink.collectedTuples.size());

  Aggregate aggregate = sink.collectedTuples.get(0);

  GPOMutable key = aggregate.getKeys();

  Assert.assertEquals(aae.timeBucket, key.getFieldInt(DimensionsDescriptor.DIMENSION_TIME_BUCKET));
  Assert.assertEquals(aae.time, key.getFieldLong(DimensionsDescriptor.DIMENSION_TIME));
  Assert.assertEquals(aae.publisher, key.getFieldString(InputItemGenerator.PUBLISHER));
  Assert.assertEquals(aae.advertiser, key.getFieldString(InputItemGenerator.ADVERTISER));
  Assert.assertEquals(aae.location, key.getFieldString(InputItemGenerator.LOCATION));

  GPOMutable aggregates = aggregate.getAggregates();

  Assert.assertEquals(aae.impressions, aggregates.getFieldLong(InputItemGenerator.IMPRESSIONS));
  Assert.assertEquals(aae.clicks, aggregates.getFieldLong(InputItemGenerator.CLICKS));
  Assert.assertEquals(aae.revenue, aggregates.getFieldDouble(InputItemGenerator.REVENUE));
  Assert.assertEquals(aae.cost, aggregates.getFieldDouble(InputItemGenerator.COST));

  sink.collectedTuples.clear();

  aae.setDimensionsDescriptorID(2);

  adsConverter.beginWindow(1L);
  adsConverter.inputPort.put(aae);
  adsConverter.endWindow();

  Assert.assertEquals(1, sink.collectedTuples.size());
  aggregate = sink.collectedTuples.get(0);

  key = aggregate.getKeys();

  Assert.assertEquals(aae.timeBucket, key.getFieldInt(DimensionsDescriptor.DIMENSION_TIME_BUCKET));
  Assert.assertEquals(aae.time, key.getFieldLong(DimensionsDescriptor.DIMENSION_TIME));
  Assert.assertFalse(key.getFieldDescriptor().getFields().getFields().contains(InputItemGenerator.PUBLISHER));
  Assert.assertEquals(aae.advertiser, key.getFieldString(InputItemGenerator.ADVERTISER));
  Assert.assertFalse(key.getFieldDescriptor().getFields().getFields().contains(InputItemGenerator.LOCATION));

  aggregates = aggregate.getAggregates();

  Assert.assertEquals(aae.impressions, aggregates.getFieldLong(InputItemGenerator.IMPRESSIONS));
  Assert.assertEquals(aae.clicks, aggregates.getFieldLong(InputItemGenerator.CLICKS));
  Assert.assertEquals(aae.revenue, aggregates.getFieldDouble(InputItemGenerator.REVENUE));
  Assert.assertEquals(aae.cost, aggregates.getFieldDouble(InputItemGenerator.COST));
}
 

@Override
public void aggregate(Aggregate dest, Aggregate src)
{
  GPOMutable destAggs = dest.getAggregates();
  GPOMutable srcAggs = src.getAggregates();

  {
    byte[] destByte = destAggs.getFieldsByte();
    if (destByte != null) {
      byte[] srcByte = srcAggs.getFieldsByte();

      for (int index = 0;
          index < destByte.length;
          index++) {
        if (destByte[index] < srcByte[index]) {
          destByte[index] = srcByte[index];
        }
      }
    }
  }

  {
    short[] destShort = destAggs.getFieldsShort();
    if (destShort != null) {
      short[] srcShort = srcAggs.getFieldsShort();

      for (int index = 0;
          index < destShort.length;
          index++) {
        if (destShort[index] < srcShort[index]) {
          destShort[index] = srcShort[index];
        }
      }
    }
  }

  {
    int[] destInteger = destAggs.getFieldsInteger();
    if (destInteger != null) {
      int[] srcInteger = srcAggs.getFieldsInteger();

      for (int index = 0;
          index < destInteger.length;
          index++) {
        if (destInteger[index] < srcInteger[index]) {
          destInteger[index] = srcInteger[index];
        }
      }
    }
  }

  {
    long[] destLong = destAggs.getFieldsLong();
    if (destLong != null) {
      long[] srcLong = srcAggs.getFieldsLong();

      for (int index = 0;
          index < destLong.length;
          index++) {
        if (destLong[index] < srcLong[index]) {
          destLong[index] = srcLong[index];
        }
      }
    }
  }

  {
    float[] destFloat = destAggs.getFieldsFloat();
    if (destFloat != null) {
      float[] srcFloat = srcAggs.getFieldsFloat();

      for (int index = 0;
          index < destFloat.length;
          index++) {
        if (destFloat[index] < srcFloat[index]) {
          destFloat[index] = srcFloat[index];
        }
      }
    }
  }

  {
    double[] destDouble = destAggs.getFieldsDouble();
    if (destDouble != null) {
      double[] srcDouble = srcAggs.getFieldsDouble();

      for (int index = 0;
          index < destDouble.length;
          index++) {
        if (destDouble[index] < srcDouble[index]) {
          destDouble[index] = srcDouble[index];
        }
      }
    }
  }
}
 

@Override
public void aggregate(Aggregate dest, Aggregate src)
{
  GPOMutable destAggs = dest.getAggregates();
  GPOMutable srcAggs = src.getAggregates();

  {
    byte[] destByte = destAggs.getFieldsByte();
    if (destByte != null) {
      byte[] srcByte = srcAggs.getFieldsByte();

      for (int index = 0;
          index < destByte.length;
          index++) {
        if (destByte[index] > srcByte[index]) {
          destByte[index] = srcByte[index];
        }
      }
    }
  }

  {
    short[] destShort = destAggs.getFieldsShort();
    if (destShort != null) {
      short[] srcShort = srcAggs.getFieldsShort();

      for (int index = 0;
          index < destShort.length;
          index++) {
        if (destShort[index] > srcShort[index]) {
          destShort[index] = srcShort[index];
        }
      }
    }
  }

  {
    int[] destInteger = destAggs.getFieldsInteger();
    if (destInteger != null) {
      int[] srcInteger = srcAggs.getFieldsInteger();

      for (int index = 0;
          index < destInteger.length;
          index++) {
        if (destInteger[index] > srcInteger[index]) {
          destInteger[index] = srcInteger[index];
        }
      }
    }
  }

  {
    long[] destLong = destAggs.getFieldsLong();
    if (destLong != null) {
      long[] srcLong = srcAggs.getFieldsLong();

      for (int index = 0;
          index < destLong.length;
          index++) {
        if (destLong[index] > srcLong[index]) {
          destLong[index] = srcLong[index];
        }
      }
    }
  }

  {
    float[] destFloat = destAggs.getFieldsFloat();
    if (destFloat != null) {
      float[] srcFloat = srcAggs.getFieldsFloat();

      for (int index = 0;
          index < destFloat.length;
          index++) {
        if (destFloat[index] > srcFloat[index]) {
          destFloat[index] = srcFloat[index];
        }
      }
    }
  }

  {
    double[] destDouble = destAggs.getFieldsDouble();
    if (destDouble != null) {
      double[] srcDouble = srcAggs.getFieldsDouble();

      for (int index = 0;
          index < destDouble.length;
          index++) {
        if (destDouble[index] > srcDouble[index]) {
          destDouble[index] = srcDouble[index];
        }
      }
    }
  }
}