org.apache.apex.malhar.lib.dimensions.DimensionsEvent.InputEvent Java Examples

@Override
public Aggregate getGroup(InputEvent src, int aggregatorIndex)
{
  src.used = true;
  GPOMutable aggregates = new GPOMutable(context.aggregateDescriptor);
  GPOMutable keys = new GPOMutable(context.keyDescriptor);
  GPOUtils.indirectCopy(keys, src.getKeys(), context.indexSubsetKeys);

  EventKey eventKey = createEventKey(src,
      context,
      aggregatorIndex);

  long[] longFields = aggregates.getFieldsLong();

  for (int index = 0;
      index < longFields.length;
      index++) {
    longFields[index] = 0;
  }

  return new Aggregate(eventKey,
      aggregates);
}
 

@Override
public Aggregate getGroup(InputEvent src, int aggregatorIndex)
{
  src.used = true;
  int[] stringIndexSubset = this.context.indexSubsetKeys.fieldsStringIndexSubset;

  String[] keys;

  if (stringIndexSubset == null) {
    keys = new String[0];
  } else {
    keys = new String[stringIndexSubset.length];
  }

  for (int counter = 0; counter < keys.length; counter++) {
    keys[counter] = src.getKeys().getFieldsString()[stringIndexSubset[counter]];
  }

  MachineAggregate machineAggregate = new MachineAggregate(keys, 0, context.schemaID, context.dimensionsDescriptorID,
      context.aggregatorID, 0L, 0L, 0L,
      this.context.dd.getCustomTimeBucket().roundDown(src.getKeys().getFieldsLong()[0]),
      this.context.customTimeBucketRegistry.getTimeBucketId(this.context.dd.getCustomTimeBucket()));

  machineAggregate.setAggregatorIndex(aggregatorIndex);
  return machineAggregate;
}
 

/**
 * Creates an {@link EventKey} from the given {@link InputEvent}.
 *
 * @param inputEvent      The {@link InputEvent} to extract an {@link EventKey} from.
 * @param context         The conversion context required to extract the {@link EventKey} from
 *                        the given {@link InputEvent}.
 * @param aggregatorIndex The aggregatorIndex to assign to this {@link InputEvent}.
 * @return The {@link EventKey} extracted from the given {@link InputEvent}.
 */
public static EventKey createEventKey(InputEvent inputEvent,
    DimensionsConversionContext context,
    int aggregatorIndex)
{
  GPOMutable keys = new GPOMutable(context.keyDescriptor);
  GPOUtils.indirectCopy(keys, inputEvent.getKeys(), context.indexSubsetKeys);

  if (context.outputTimebucketIndex >= 0) {
    CustomTimeBucket timeBucket = context.dd.getCustomTimeBucket();

    keys.getFieldsInteger()[context.outputTimebucketIndex] = context.customTimeBucketRegistry.getTimeBucketId(
        timeBucket);
    keys.getFieldsLong()[context.outputTimestampIndex] =
        timeBucket.roundDown(inputEvent.getKeys().getFieldsLong()[context.inputTimestampIndex]);
  }

  EventKey eventKey = new EventKey(context.schemaID,
      context.dimensionsDescriptorID,
      context.aggregatorID,
      keys);

  return eventKey;
}
 

/**
 * Creates an {@link Aggregate} from the given {@link InputEvent}.
 *
 * @param inputEvent      The {@link InputEvent} to unpack into an {@link Aggregate}.
 * @param context         The conversion context required to transform the {@link InputEvent} into
 *                        the correct {@link Aggregate}.
 * @param aggregatorIndex The aggregatorIndex assigned to this {@link Aggregate}.
 * @return The converted {@link Aggregate}.
 */
public static Aggregate createAggregate(InputEvent inputEvent,
    DimensionsConversionContext context,
    int aggregatorIndex)
{
  GPOMutable aggregates = new GPOMutable(context.aggregateDescriptor);
  EventKey eventKey = createEventKey(inputEvent,
      context,
      aggregatorIndex);

  Aggregate aggregate = new Aggregate(eventKey,
      aggregates);
  aggregate.setAggregatorIndex(aggregatorIndex);

  return aggregate;
}
 

@Override
public Aggregate getGroup(InputEvent src, int aggregatorIndex)
{
  Aggregate aggregate = super.getGroup(src, aggregatorIndex);

  GPOUtils.indirectCopy(aggregate.getAggregates(), src.getAggregates(), context.indexSubsetAggregates);

  return aggregate;
}
 

@Override
public void aggregate(Aggregate dest, InputEvent src)
{
  long[] fieldsLong = dest.getAggregates().getFieldsLong();

  for (int index = 0;
      index < fieldsLong.length;
      index++) {
    //increment count
    fieldsLong[index]++;
  }
}
 

@Override
public void aggregate(Aggregate dest, InputEvent src)
{
  GPOMutable destAggs = dest.getAggregates();
  GPOMutable srcAggs = src.getAggregates();

  aggregateInput(destAggs, srcAggs);
}
 

@Override
public Aggregate getGroup(InputEvent src, int aggregatorIndex)
{
  src.used = true;
  Aggregate aggregate = createAggregate(src,
      context,
      aggregatorIndex);

  GPOMutable value = aggregate.getAggregates();
  GPOUtils.zeroFillNumeric(value);

  return aggregate;
}
 

@Override
public Aggregate getGroup(InputEvent src, int aggregatorIndex)
{
  Aggregate aggregate = super.getGroup(src, aggregatorIndex);

  GPOUtils.indirectCopy(aggregate.getAggregates(), src.getAggregates(), context.indexSubsetAggregates);

  return aggregate;
}
 

@Override
public Aggregate getGroup(InputEvent src, int aggregatorIndex)
{
  Aggregate aggregate = super.getGroup(src, aggregatorIndex);

  GPOUtils.indirectCopy(aggregate.getAggregates(), src.getAggregates(), context.indexSubsetAggregates);

  return aggregate;
}
 

@Override
public Aggregate getGroup(InputEvent src, int aggregatorIndex)
{
  Aggregate aggregate = super.getGroup(src, aggregatorIndex);

  GPOUtils.indirectCopy(aggregate.getAggregates(), src.getAggregates(), context.indexSubsetAggregates);

  return aggregate;
}
 

@Override
public void aggregate(Aggregate dest, InputEvent src)
{
  @SuppressWarnings("unchecked")
  List<GPOMutable> destKeys =
      (List<GPOMutable>)dest.getMetaData().getFieldsObject()[KEYS_INDEX];

  @SuppressWarnings("unchecked")
  List<GPOMutable> destAggregates =
      (List<GPOMutable>)dest.getMetaData().getFieldsObject()[AGGREGATES_INDEX];

  long timestamp = 0L;

  if (context.inputTimestampIndex >= 0) {
    timestamp = src.getKeys().getFieldsLong()[context.inputTimestampIndex];
    src.getKeys().getFieldsLong()[context.inputTimestampIndex] = -1L;
  }

  if (!contains(destKeys, src.getKeys())) {
    destKeys.add(new GPOMutable(src.getKeys()));

    GPOMutable aggregates = new GPOMutable(context.aggregateDescriptor);
    GPOUtils.indirectCopy(aggregates, src.getAggregates(), context.indexSubsetAggregates);

    destAggregates.add(aggregates);

    this.aggregateAggs(dest.getAggregates(), aggregates);
  }

  if (context.inputTimestampIndex >= 0) {
    src.getKeys().getFieldsLong()[context.inputTimestampIndex] = timestamp;
  }
}
 

@Override
public Aggregate getGroup(InputEvent src, int aggregatorIndex)
{
  src.used = true;
  Aggregate agg = createAggregate(src,
      context,
      aggregatorIndex);

  GPOUtils.indirectCopy(agg.getAggregates(), src.getAggregates(), context.indexSubsetAggregates);

  GPOMutable metaData = new GPOMutable(getMetaDataDescriptor());

  GPOMutable fullKey = new GPOMutable(src.getKeys());

  if (context.inputTimestampIndex >= 0) {
    fullKey.getFieldsLong()[context.inputTimestampIndex] = -1L;
  }

  List<GPOMutable> keys = Lists.newArrayList(fullKey);

  GPOMutable value = new GPOMutable(agg.getAggregates());
  List<GPOMutable> values = Lists.newArrayList(value);

  metaData.getFieldsObject()[KEY_FD_INDEX] = fullKey.getFieldDescriptor();
  metaData.getFieldsObject()[AGGREGATE_FD_INDEX] = value.getFieldDescriptor();
  metaData.getFieldsObject()[KEYS_INDEX] = keys;
  metaData.getFieldsObject()[AGGREGATES_INDEX] = values;
  agg.setMetaData(metaData);

  return agg;
}
 

@Override
public Aggregate getGroup(InputEvent src, int aggregatorIndex)
{
  src.used = true;
  Aggregate aggregate = createAggregate(src,
      context,
      aggregatorIndex);
  return aggregate;
}
 

@Override
public void aggregate(Aggregate dest, InputEvent src)
{
  ((MachineAggregate)dest).cpuUsage += src.getAggregates().getFieldsLong()[0];
  ((MachineAggregate)dest).hddUsage += src.getAggregates().getFieldsLong()[1];
  ((MachineAggregate)dest).ramUsage += src.getAggregates().getFieldsLong()[2];
}
 

@Override
public Aggregate getGroup(InputEvent src, int aggregatorIndex)
{
  src.used = true;
  int[] stringIndexSubset = this.context.indexSubsetKeys.fieldsStringIndexSubset;

  String[] keys;

  if (stringIndexSubset == null) {
    keys = new String[0];
  } else {
    keys = new String[stringIndexSubset.length];
  }

  for (int counter = 0; counter < keys.length; counter++) {
    keys[counter] = src.getKeys().getFieldsString()[stringIndexSubset[counter]];
  }

  MachineAggregate machineAggregate = new MachineAggregate(keys, 0, context.schemaID, context.dimensionsDescriptorID,
      context.aggregatorID, src.getAggregates().getFieldsLong()[0], src.getAggregates().getFieldsLong()[2],
      src.getAggregates().getFieldsLong()[1],
      this.context.dd.getCustomTimeBucket().roundDown(src.getEventKey().getKey().getFieldsLong()[0]),
      this.context.customTimeBucketRegistry.getTimeBucketId(this.context.dd.getCustomTimeBucket()));

  machineAggregate.setAggregatorIndex(aggregatorIndex);
  return machineAggregate;
}
 

@Override
public void aggregate(Aggregate dest, InputEvent src)
{
  ((MachineAggregate)dest).cpuUsage++;
  ((MachineAggregate)dest).hddUsage++;
  ((MachineAggregate)dest).ramUsage++;
}
 

@Override
public void populateDAG(DAG dag, Configuration conf)
{
  InputItemGenerator input = dag.addOperator("InputGenerator", InputItemGenerator.class);
  DimensionsComputationFlexibleSingleSchemaPOJO dimensions = dag.addOperator("DimensionsComputation",
      DimensionsComputationFlexibleSingleSchemaPOJO.class);
  dag.getMeta(dimensions).getAttributes().put(Context.OperatorContext.APPLICATION_WINDOW_COUNT, 10);
  DevNull<Object> devNull = dag.addOperator("DevNull", new DevNull<Object>());

  //Set input properties
  String eventSchema = SchemaUtils.jarResourceFileToString("adsBenchmarkSchema.json");
  input.setEventSchemaJSON(eventSchema);

  Map<String, String> keyToExpression = Maps.newHashMap();
  keyToExpression.put("publisher", "getPublisher()");
  keyToExpression.put("advertiser", "getAdvertiser()");
  keyToExpression.put("location", "getLocation()");
  keyToExpression.put("time", "getTime()");

  Map<String, String> aggregateToExpression = Maps.newHashMap();
  aggregateToExpression.put("cost", "getCost()");
  aggregateToExpression.put("revenue", "getRevenue()");
  aggregateToExpression.put("impressions", "getImpressions()");
  aggregateToExpression.put("clicks", "getClicks()");

  DimensionsComputationUnifierImpl<InputEvent, Aggregate> unifier = new DimensionsComputationUnifierImpl<InputEvent, Aggregate>();
  dimensions.setUnifier(unifier);
  dimensions.setKeyToExpression(keyToExpression);
  dimensions.setAggregateToExpression(aggregateToExpression);
  dimensions.setConfigurationSchemaJSON(eventSchema);

  dag.addStream("InputStream", input.outputPort, dimensions.input).setLocality(Locality.CONTAINER_LOCAL);
  dag.addStream("DimensionalData", dimensions.output, devNull.data);
}
 

public void populateDimensionsDAG(DAG dag, Configuration conf, DefaultOutputPort<DimensionTuple> upstreamPort) 
{
  final String eventSchema = SchemaUtils.jarResourceFileToString(eventSchemaLocation);
  
  // dimension
  DimensionsComputationFlexibleSingleSchemaPOJO dimensions = dag.addOperator("DimensionsComputation",
      DimensionsComputationFlexibleSingleSchemaPOJO.class);

  // Set operator properties
  // key expression
  {
    Map<String, String> keyToExpression = Maps.newHashMap();
    keyToExpression.put("campaignId", DimensionTuple.CAMPAIGNID);
    keyToExpression.put("time", DimensionTuple.EVENTTIME);
    dimensions.setKeyToExpression(keyToExpression);
  }

  // aggregate expression
  {
    Map<String, String> valueToExpression = Maps.newHashMap();
    valueToExpression.put("clicks", DimensionTuple.CLICKS);
    valueToExpression.put("latency", DimensionTuple.LATENCY);
    
    dimensions.setAggregateToExpression(valueToExpression);
  }

  // event schema
  dimensions.setConfigurationSchemaJSON(eventSchema);
  dimensions.setUnifier(new DimensionsComputationUnifierImpl<InputEvent, Aggregate>());

  dag.setUnifierAttribute(dimensions.output, OperatorContext.MEMORY_MB, 10240);
  
  dag.setInputPortAttribute(dimensions.input, Context.PortContext.PARTITION_PARALLEL, true);
  
  // store
  AppDataSingleSchemaDimensionStoreHDHT store = createStore(dag, conf, eventSchema); 
  store.setCacheWindowDuration(10000 * 5 / STREAMING_WINDOW_SIZE_MILLIS);   //cache for 5 windows
  dag.addStream("GenerateStream", upstreamPort, dimensions.input).setLocality(Locality.CONTAINER_LOCAL);
  
  StoreStreamCodec codec = new StoreStreamCodec();
  dag.setInputPortAttribute(store.input, PortContext.STREAM_CODEC, codec);
  dag.addStream("DimensionalStream", dimensions.output, store.input);

  
  if (includeQuery) {
    createQuery(dag, conf, store);

    // wsOut
    PubSubWebSocketAppDataResult wsOut = createQueryResult(dag, conf, store);

    dag.addStream("QueryResult", store.queryResult, wsOut.input);
  } else {
    DevNull devNull = new DevNull();
    dag.addOperator("devNull", devNull);
    dag.addStream("QueryResult", store.queryResult, devNull.data);
  }
  
  dag.setAttribute(DAGContext.STREAMING_WINDOW_SIZE_MILLIS, STREAMING_WINDOW_SIZE_MILLIS);
}
 

@Override
public void populateDAG(DAG dag, Configuration conf)
{
  String propStorePath = "dt.application." + APP_NAME + ".operator.Store.fileStore.basePathPrefix";

  String eventSchema = SchemaUtils.jarResourceFileToString(EVENT_SCHEMA);
  InputReceiver randomGen = dag.addOperator("Receiver", InputReceiver.class);
  randomGen.setEventSchema(eventSchema);

  DimensionsComputationFlexibleSingleSchemaPOJO dimensions = dag.addOperator("DimensionsComputation", getDimensions());
  dag.getMeta(dimensions).getAttributes().put(Context.OperatorContext.APPLICATION_WINDOW_COUNT, 6);
  dag.getMeta(dimensions).getAttributes().put(Context.OperatorContext.CHECKPOINT_WINDOW_COUNT, 6);
  AppDataSingleSchemaDimensionStoreHDHT store = dag.addOperator("Store", getStore());
  store.setCacheWindowDuration(120);
  @SuppressWarnings("unchecked")
  PassThroughOperator unifier = dag.addOperator("Unifier", PassThroughOperator.class);

  //Set operator properties

  Map<String, String> keyToExpression = Maps.newHashMap();
  keyToExpression.put("Customer ID", "getMachineKey().getCustomer()");
  keyToExpression.put("Product ID", "getMachineKey().getProduct()");
  keyToExpression.put("Product OS", "getMachineKey().getOs()");
  keyToExpression.put("Software1 Ver", "getMachineKey().getSoftware1()");
  keyToExpression.put("Software2 Ver", "getMachineKey().getSoftware2()");
  keyToExpression.put("Device ID", "getMachineKey().getDeviceId()");
  keyToExpression.put("time", "getMachineKey().getTimestamp()");

  Map<String, String> aggregateToExpression = Maps.newHashMap();
  aggregateToExpression.put("CPU Usage (%)", "getCpu()");
  aggregateToExpression.put("RAM Usage (%)", "getRam()");
  aggregateToExpression.put("HDD Usage (%)", "getHdd()");

  dimensions.setKeyToExpression(keyToExpression);
  dimensions.setAggregateToExpression(aggregateToExpression);
  dimensions.setConfigurationSchemaJSON(eventSchema);

  dimensions.setUnifier(new DimensionsComputationUnifierImpl<InputEvent, Aggregate>());
  dag.getMeta(dimensions).getMeta(dimensions.output).getUnifierMeta().getAttributes().put(OperatorContext.MEMORY_MB, 8092);

  //Set store properties
  String basePath = Preconditions.checkNotNull(conf.get(propStorePath),
      "a base path should be specified in the properties.xml");
  TFileImpl hdsFile = new TFileImpl.DTFileImpl();
  basePath += Path.SEPARATOR + System.currentTimeMillis();
  hdsFile.setBasePath(basePath);
  store.setFileStore(hdsFile);
  store.getResultFormatter().setContinuousFormatString("#.00");
  store.setConfigurationSchemaJSON(eventSchema);
  store.setQueryResultUnifier(new DimensionStoreHDHTNonEmptyQueryResultUnifier());

  store.setEmbeddableQueryInfoProvider(new PubSubWebSocketAppDataQuery());
  PubSubWebSocketAppDataResult wsOut = dag.addOperator("QueryResult", new PubSubWebSocketAppDataResult());

  //Set remaining dag options

  dag.addStream("InputStream", randomGen.outputInline, dimensions.input).setLocality(Locality.CONTAINER_LOCAL);
  dag.addStream("DimensionalData", dimensions.output, unifier.input);
  dag.addStream("Unifier", unifier.output, store.input);
  dag.addStream("QueryResult", store.queryResult, wsOut.input);
}
 

@Override
public void aggregate(Aggregate dest, InputEvent src)
{
  GPOUtils.indirectCopy(dest.getAggregates(), src.getAggregates(), context.indexSubsetAggregates);
}
 

@Override
public void aggregate(Aggregate dest, InputEvent src)
{
  //Ignore
}
 

@Override
public void populateDAG(DAG dag, Configuration conf)
{
  String propStorePath = "dt.application." + appName + ".operator.Store.fileStore.basePathPrefix";

  //Declare operators

  //Set input properties
  String eventSchema = SchemaUtils.jarResourceFileToString(eventSchemaLocation);

  if (inputOperator == null) {
    InputItemGenerator input = dag.addOperator("InputGenerator", InputItemGenerator.class);
    input.advertiserName = advertisers;
    input.setEventSchemaJSON(eventSchema);
    inputOperator = input;
  } else {
    dag.addOperator("InputGenerator", inputOperator);
  }

  DimensionsComputationFlexibleSingleSchemaPOJO dimensions = dag.addOperator("DimensionsComputation", DimensionsComputationFlexibleSingleSchemaPOJO.class);
  dag.getMeta(dimensions).getAttributes().put(Context.OperatorContext.APPLICATION_WINDOW_COUNT, 4);
  dag.getMeta(dimensions).getAttributes().put(Context.OperatorContext.CHECKPOINT_WINDOW_COUNT, 4);
  AppDataSingleSchemaDimensionStoreHDHT store = dag.addOperator("Store", AppDataSingleSchemaDimensionStoreHDHT.class);

  //Set operator properties

  Map<String, String> keyToExpression = Maps.newHashMap();
  keyToExpression.put("publisher", "getPublisher()");
  keyToExpression.put("advertiser", "getAdvertiser()");
  keyToExpression.put("location", "getLocation()");
  keyToExpression.put("time", "getTime()");

  Map<String, String> aggregateToExpression = Maps.newHashMap();
  aggregateToExpression.put("cost", "getCost()");
  aggregateToExpression.put("revenue", "getRevenue()");
  aggregateToExpression.put("impressions", "getImpressions()");
  aggregateToExpression.put("clicks", "getClicks()");

  dimensions.setKeyToExpression(keyToExpression);
  dimensions.setAggregateToExpression(aggregateToExpression);
  dimensions.setConfigurationSchemaJSON(eventSchema);

  dimensions.setUnifier(new DimensionsComputationUnifierImpl<InputEvent, Aggregate>());
  dag.getMeta(dimensions).getMeta(dimensions.output).getUnifierMeta().getAttributes().put(OperatorContext.MEMORY_MB, 8092);

  //Set store properties
  String basePath = Preconditions.checkNotNull(conf.get(propStorePath),
      "a base path should be specified in the properties.xml");
  TFileImpl hdsFile = new TFileImpl.DTFileImpl();
  basePath += Path.SEPARATOR + System.currentTimeMillis();
  hdsFile.setBasePath(basePath);
  store.setFileStore(hdsFile);
  store.getResultFormatter().setContinuousFormatString("#.00");
  store.setConfigurationSchemaJSON(eventSchema);

  store.setEmbeddableQueryInfoProvider(new PubSubWebSocketAppDataQuery());
  PubSubWebSocketAppDataResult wsOut = dag.addOperator("QueryResult", new PubSubWebSocketAppDataResult());

  //Set remaining dag options

  dag.setAttribute(store, Context.OperatorContext.COUNTERS_AGGREGATOR, new BasicCounters.LongAggregator<MutableLong>());

  dag.addStream("InputStream", inputOperator.getOutputPort(), dimensions.input).setLocality(Locality.CONTAINER_LOCAL);
  dag.addStream("DimensionalData", dimensions.output, store.input);
  dag.addStream("QueryResult", store.queryResult, wsOut.input);
}
 

@Override
public void populateDAG(DAG dag, Configuration conf)
{
  //Set input properties
  String eventSchema = SchemaUtils.jarResourceFileToString(eventSchemaLocation);

  //input
  if (inputOperator == null) {
    inputOperator = new EnrichedCDRHbaseInputOperator();
  }
  dag.addOperator("InputGenerator", inputOperator);

  //dimension
  DimensionsComputationFlexibleSingleSchemaPOJO dimensions = dag.addOperator("DimensionsComputation",
      DimensionsComputationFlexibleSingleSchemaPOJO.class);
  dag.getMeta(dimensions).getAttributes().put(Context.OperatorContext.APPLICATION_WINDOW_COUNT, 4);
  dag.getMeta(dimensions).getAttributes().put(Context.OperatorContext.CHECKPOINT_WINDOW_COUNT, 4);

  //Set operator properties
  //key expression
  {
    Map<String, String> keyToExpression = Maps.newHashMap();
    keyToExpression.put("imsi", "getImsi()");
    keyToExpression.put("Carrier", "getOperatorCode()");
    keyToExpression.put("imei", "getImei()");
    dimensions.setKeyToExpression(keyToExpression);
  }

  EnrichedCDR cdr = new EnrichedCDR();
  cdr.getOperatorCode();
  cdr.getDuration();

  //aggregate expression
  {
    Map<String, String> aggregateToExpression = Maps.newHashMap();
    aggregateToExpression.put("duration", "getDuration()");
    aggregateToExpression.put("terminatedAbnomally", "getTerminatedAbnomally()");
    aggregateToExpression.put("terminatedNomally", "getTerminatedNomally()");
    aggregateToExpression.put("called", "getCalled()");
    dimensions.setAggregateToExpression(aggregateToExpression);
  }

  //event schema
  dimensions.setConfigurationSchemaJSON(eventSchema);

  dimensions.setUnifier(new DimensionsComputationUnifierImpl<InputEvent, Aggregate>());
  dag.getMeta(dimensions).getMeta(dimensions.output).getUnifierMeta().getAttributes().put(OperatorContext.MEMORY_MB,
      8092);

  //store
  AppDataSingleSchemaDimensionStoreHDHT store = dag.addOperator("Store", AppDataSingleSchemaDimensionStoreHDHT.class);
  String basePath = conf.get(PROP_STORE_PATH);
  if (basePath == null || basePath.isEmpty()) {
    basePath = Preconditions.checkNotNull(conf.get(PROP_STORE_PATH),
        "base path should be specified in the properties.xml");
  }
  TFileImpl hdsFile = new TFileImpl.DTFileImpl();
  basePath += System.currentTimeMillis();
  hdsFile.setBasePath(basePath);

  store.setFileStore(hdsFile);
  dag.setAttribute(store, Context.OperatorContext.COUNTERS_AGGREGATOR,
      new BasicCounters.LongAggregator<MutableLong>());
  store.setConfigurationSchemaJSON(eventSchema);
  //store.setDimensionalSchemaStubJSON(eventSchema);

  PubSubWebSocketAppDataQuery query = createAppDataQuery();
  store.setEmbeddableQueryInfoProvider(query);

  //wsOut
  PubSubWebSocketAppDataResult wsOut = createAppDataResult();
  dag.addOperator("QueryResult", wsOut);
  //Set remaining dag options

  dag.setAttribute(store, Context.OperatorContext.COUNTERS_AGGREGATOR,
      new BasicCounters.LongAggregator<MutableLong>());

  dag.addStream("InputStream", inputOperator.outputPort, dimensions.input).setLocality(Locality.CONTAINER_LOCAL);
  dag.addStream("DimensionalData", dimensions.output, store.input);
  dag.addStream("QueryResult", store.queryResult, wsOut.input);
}
 

protected void populateCsGeoDAG(DAG dag, Configuration conf,
    List<DefaultInputPort<? super EnrichedCustomerService>> customerServiceStreamSinks)
{
  // dimension
  DimensionsComputationFlexibleSingleSchemaPOJO dimensions = dag.addOperator("TagServiceGeoLocations",
      DimensionsComputationFlexibleSingleSchemaPOJO.class);
  dag.getMeta(dimensions).getAttributes().put(Context.OperatorContext.APPLICATION_WINDOW_COUNT, 4);
  dag.getMeta(dimensions).getAttributes().put(Context.OperatorContext.CHECKPOINT_WINDOW_COUNT, 4);

  customerServiceStreamSinks.add(dimensions.input);

  // Set operator properties
  // key expression: Point( Lat, Lon )
  {
    Map<String, String> keyToExpression = Maps.newHashMap();
    keyToExpression.put("zipcode", "getZipCode()");
    keyToExpression.put("region", "getRegionZip2()");
    keyToExpression.put("time", "getTime()");
    dimensions.setKeyToExpression(keyToExpression);
  }

  // aggregate expression: disconnect and downloads
  {
    Map<String, String> aggregateToExpression = Maps.newHashMap();
    aggregateToExpression.put("wait", "getWait()");
    aggregateToExpression.put("lat", "getLat()");
    aggregateToExpression.put("lon", "getLon()");
    dimensions.setAggregateToExpression(aggregateToExpression);
  }

  // event schema
  String geoSchema = SchemaUtils.jarResourceFileToString(csGeoSchemaLocation);
  dimensions.setConfigurationSchemaJSON(geoSchema);

  dimensions.setUnifier(new DimensionsComputationUnifierImpl<InputEvent, Aggregate>());
  dag.getMeta(dimensions).getMeta(dimensions.output).getUnifierMeta().getAttributes().put(OperatorContext.MEMORY_MB,
      8092);

  // store
  //AppDataSingleSchemaDimensionStoreHDHT store = dag.addOperator("StoreTaggedServiceGeoLocations", AppDataSingleSchemaDimensionStoreHDHT.class);
  GeoDimensionStore store = dag.addOperator("StoreTaggedServiceGeoLocations", GeoDimensionStore.class);
  store.setUpdateEnumValues(true);
  String basePath = Preconditions.checkNotNull(conf.get(PROP_GEO_STORE_PATH),
      "GEO base path should be specified in the properties.xml");
  TFileImpl hdsFile = new TFileImpl.DTFileImpl();
  basePath += System.currentTimeMillis();
  hdsFile.setBasePath(basePath);

  store.setFileStore(hdsFile);
  store.setConfigurationSchemaJSON(geoSchema);
  dag.setAttribute(store, Context.OperatorContext.COUNTERS_AGGREGATOR,
      new BasicCounters.LongAggregator<MutableLong>());

  PubSubWebSocketAppDataQuery query = createAppDataQuery();
  URI queryUri = ConfigUtil.getAppDataQueryPubSubURI(dag, conf);
  query.setUri(queryUri);
  store.setEmbeddableQueryInfoProvider(query);
  if (csGeoStorePartitionCount > 1) {
    store.setPartitionCount(csGeoStorePartitionCount);
    store.setQueryResultUnifier(new DimensionStoreHDHTNonEmptyQueryResultUnifier());
  }

  // wsOut
  PubSubWebSocketAppDataResult wsOut = createAppDataResult();
  wsOut.setUri(queryUri);
  dag.addOperator("CSGeoQueryResult", wsOut);
  // Set remaining dag options

  dag.setAttribute(store, Context.OperatorContext.COUNTERS_AGGREGATOR,
      new BasicCounters.LongAggregator<MutableLong>());

  dag.addStream("CSGeoStream", dimensions.output, store.input);
  dag.addStream("CSGeoQueryResult", store.queryResult, wsOut.input);
}
 

protected void populateCdrGeoDAG(DAG dag, Configuration conf,
    List<DefaultInputPort<? super EnrichedCDR>> enrichedStreamSinks)
{
  // dimension
  DimensionsComputationFlexibleSingleSchemaPOJO dimensions = dag.addOperator("TagNetworkGeoLocations",
      DimensionsComputationFlexibleSingleSchemaPOJO.class);
  dag.getMeta(dimensions).getAttributes().put(Context.OperatorContext.APPLICATION_WINDOW_COUNT, 4);
  dag.getMeta(dimensions).getAttributes().put(Context.OperatorContext.CHECKPOINT_WINDOW_COUNT, 4);

  enrichedStreamSinks.add(dimensions.input);

  // Set operator properties
  // key expression: Point( Lat, Lon )
  {
    Map<String, String> keyToExpression = Maps.newHashMap();
    keyToExpression.put("zipcode", "getZipCode()");
    keyToExpression.put("region", "getRegionZip2()");
    keyToExpression.put("time", "getTime()");
    dimensions.setKeyToExpression(keyToExpression);
  }

  // aggregate expression: disconnect and downloads
  {
    Map<String, String> aggregateToExpression = Maps.newHashMap();
    aggregateToExpression.put("disconnectCount", "getDisconnectCount()");
    aggregateToExpression.put("downloadBytes", "getBytes()");
    aggregateToExpression.put("lat", "getLat()");
    aggregateToExpression.put("lon", "getLon()");
    dimensions.setAggregateToExpression(aggregateToExpression);
  }

  // event schema
  String cdrGeoSchema = SchemaUtils.jarResourceFileToString(cdrGeoSchemaLocation);
  dimensions.setConfigurationSchemaJSON(cdrGeoSchema);

  dimensions.setUnifier(new DimensionsComputationUnifierImpl<InputEvent, Aggregate>());
  dag.getMeta(dimensions).getMeta(dimensions.output).getUnifierMeta().getAttributes().put(OperatorContext.MEMORY_MB,
      8092);

  // store
  //AppDataSingleSchemaDimensionStoreHDHT store = dag.addOperator("StoreNetworkTaggedGeoLocations", AppDataSingleSchemaDimensionStoreHDHT.class);
  GeoDimensionStore store = dag.addOperator("StoreNetworkTaggedGeoLocations", GeoDimensionStore.class);
  store.setUpdateEnumValues(true);
  String basePath = Preconditions.checkNotNull(conf.get(PROP_GEO_STORE_PATH),
      "GEO base path should be specified in the properties.xml");
  TFileImpl hdsFile = new TFileImpl.DTFileImpl();
  basePath += System.currentTimeMillis();
  hdsFile.setBasePath(basePath);

  store.setFileStore(hdsFile);
  store.setConfigurationSchemaJSON(cdrGeoSchema);
  dag.setAttribute(store, Context.OperatorContext.COUNTERS_AGGREGATOR,
      new BasicCounters.LongAggregator<MutableLong>());

  PubSubWebSocketAppDataQuery query = createAppDataQuery();
  URI queryUri = ConfigUtil.getAppDataQueryPubSubURI(dag, conf);
  query.setUri(queryUri);
  store.setEmbeddableQueryInfoProvider(query);
  if (cdrGeoStorePartitionCount > 1) {
    store.setPartitionCount(cdrGeoStorePartitionCount);
    store.setQueryResultUnifier(new DimensionStoreHDHTNonEmptyQueryResultUnifier());
  }

  // wsOut
  PubSubWebSocketAppDataResult wsOut = createAppDataResult();
  wsOut.setUri(queryUri);
  dag.addOperator("CDRGeoQueryResult", wsOut);
  // Set remaining dag options

  dag.setAttribute(store, Context.OperatorContext.COUNTERS_AGGREGATOR,
      new BasicCounters.LongAggregator<MutableLong>());

  dag.addStream("CDRGeoStream", dimensions.output, store.input);
  dag.addStream("CDRGeoQueryResult", store.queryResult, wsOut.input);
}
 

@Override
public void populateDAG(DAG dag, Configuration conf)
{
  String eventSchema = SchemaUtils.jarResourceFileToString(EVENT_SCHEMA);

  if (inputGenerator == null) {
    JsonSalesGenerator input = dag.addOperator("InputGenerator", JsonSalesGenerator.class);
    input.setEventSchemaJSON(eventSchema);
    inputGenerator = input;
  } else {
    dag.addOperator("InputGenerator", inputGenerator);
  }

  JsonToMapConverter converter = dag.addOperator("Converter", JsonToMapConverter.class);
  EnrichmentOperator enrichmentOperator = dag.addOperator("Enrichment", EnrichmentOperator.class);
  DimensionsComputationFlexibleSingleSchemaMap dimensions = dag.addOperator("DimensionsComputation",
      DimensionsComputationFlexibleSingleSchemaMap.class);
  dag.getMeta(dimensions).getAttributes().put(Context.OperatorContext.APPLICATION_WINDOW_COUNT, 4);
  AppDataSingleSchemaDimensionStoreHDHT store = dag.addOperator("Store", AppDataSingleSchemaDimensionStoreHDHT.class);

  String basePath = conf.get(PROP_STORE_PATH);
  TFileImpl hdsFile = new TFileImpl.DTFileImpl();
  basePath += System.currentTimeMillis();
  hdsFile.setBasePath(basePath);

  store.setFileStore(hdsFile);
  dag.setAttribute(store, Context.OperatorContext.COUNTERS_AGGREGATOR,
      new BasicCounters.LongAggregator<MutableLong>());
  String dimensionalSchema = SchemaUtils.jarResourceFileToString(DIMENSIONAL_SCHEMA);

  dimensions.setConfigurationSchemaJSON(eventSchema);
  Map<String, String> fieldToMapField = Maps.newHashMap();
  dimensions.setValueNameAliases(fieldToMapField);
  dimensions.setUnifier(new DimensionsComputationUnifierImpl<InputEvent, Aggregate>());
  dag.getMeta(dimensions).getMeta(dimensions.output).getUnifierMeta().getAttributes().put(OperatorContext.MEMORY_MB,
      8092);

  store.setConfigurationSchemaJSON(eventSchema);
  store.setDimensionalSchemaStubJSON(dimensionalSchema);

  PubSubWebSocketAppDataQuery wsIn = new PubSubWebSocketAppDataQuery();
  store.setEmbeddableQueryInfoProvider(wsIn);

  PubSubWebSocketAppDataResult wsOut = dag.addOperator("QueryResult", new PubSubWebSocketAppDataResult());

  dag.addStream("InputStream", inputGenerator.getOutputPort(), converter.input);
  dag.addStream("EnrichmentStream", converter.outputMap, enrichmentOperator.inputPort);
  dag.addStream("ConvertStream", enrichmentOperator.outputPort, dimensions.input);
  dag.addStream("DimensionalData", dimensions.output, store.input);
  dag.addStream("QueryResult", store.queryResult, wsOut.input).setLocality(Locality.CONTAINER_LOCAL);
}