backtype.storm.generated.Grouping Java Examples

@SuppressWarnings("rawtypes")
@Override
void prepare(Map conf, TopologyContext context) {
	// use TTL and maxSize from config if they were not set explicitly using the constructor (implicit way of doing this...)
	if(TTL == 29) TTL = conf.get(StormCVConfig.STORMCV_CACHES_TIMEOUT_SEC) == null ? TTL : ((Long)conf.get(StormCVConfig.STORMCV_CACHES_TIMEOUT_SEC)).intValue();
	if(maxSize == 256) maxSize = conf.get(StormCVConfig.STORMCV_CACHES_MAX_SIZE) == null ? maxSize : ((Long)conf.get(StormCVConfig.STORMCV_CACHES_MAX_SIZE)).intValue();
	history = new History(this);
	
	// IF NO grouping was set THEN select the first grouping registered for the spout as the grouping used within the Spout (usually a good guess)
	if(groupBy == null){
		Map<GlobalStreamId, Grouping> sources = context.getSources(context.getThisComponentId());
		for(GlobalStreamId id : sources.keySet()){
			Grouping grouping = sources.get(id);
			this.groupBy = new Fields(grouping.get_fields());
			break;
		}
	}
	
	// prepare the selector and operation
	try {
		batcher.prepare(conf);
		operation.prepare(conf, context);
	} catch (Exception e) {
		logger.error("Unable to preapre the Selector or Operation", e);
	}
}
 

public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
  this.collector = collector;
  this.conf = stormConf;
  if (conf.containsKey("bolt.parser.num.of.key.chars.to.use.for.shuffle.grouping")) {
    this.numberOfCharsToUseForShuffleGrouping = Integer.valueOf(conf.get(
        "bolt.parser.num.of.key.chars.to.use.for.shuffle.grouping").toString());
  }
  
  Grouping._Fields a;


  // hBaseStreamPartitioner = new HBaseStreamPartitioner(
  // conf.get("bolt.hbase.table.name").toString(),
  // 0,
  // Integer.parseInt(conf.get("bolt.hbase.partitioner.region.info.refresh.interval.mins").toString()))
  // ;
  // hBaseStreamPartitioner.prepare();

}
 

/**
 * get current task's output task list
 */
public static Set<Integer> worker_output_tasks(WorkerData workerData) {
    ContextMaker context_maker = workerData.getContextMaker();
    Set<Integer> taskIds = workerData.getTaskIds();
    StormTopology topology = workerData.getSysTopology();

    Set<Integer> rtn = new HashSet<>();

    for (Integer taskId : taskIds) {
        TopologyContext context = context_maker.makeTopologyContext(topology, taskId, null);

        // <StreamId, <ComponentId, Grouping>>
        Map<String, Map<String, Grouping>> targets = context.getThisTargets();
        for (Map<String, Grouping> e : targets.values()) {
            for (String componentId : e.keySet()) {
                List<Integer> tasks = context.getComponentTasks(componentId);
                rtn.addAll(tasks);
            }
        }
    }

    return rtn;
}
 

@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
    createState(context);
    prepare(stormConf, context, collector, keyRangeState);
    
    Map<GlobalStreamId, Grouping> sources = context.getSources(context.getThisComponentId());
    for (Map.Entry<GlobalStreamId, Grouping> entry : sources.entrySet()) {
        GlobalStreamId stream = entry.getKey();
        Grouping grouping = entry.getValue();
        Grouping._Fields groupingFields = Thrift.groupingType(grouping);
        if (Grouping._Fields.FIELDS.equals(groupingFields)) {
            Fields fields = new Fields(Thrift.fieldGrouping(grouping));
            fieldGrouping.put(stream.get_streamId(), fields);
        }
    }
    LOG.info("Source fieldgrouping streams: {}", fieldGrouping);
}
 

public MkGrouper(TopologyContext _topology_context, Fields _out_fields, Grouping _thrift_grouping,
                 String targetComponent, String streamId, WorkerData workerData) {
    this.topologyContext = _topology_context;
    this.outFields = _out_fields;
    this.thriftGrouping = _thrift_grouping;
    this.streamId = streamId;
    this.targetComponent = targetComponent;

    List<Integer> outTasks = topologyContext.getComponentTasks(targetComponent);
    this.outTasks = new ArrayList<>();
    this.outTasks.addAll(outTasks);
    Collections.sort(this.outTasks);

    this.localTasks = _topology_context.getThisWorkerTasks();
    this.fields = Thrift.groupingType(thriftGrouping);
    this.groupType = this.parseGroupType(workerData);

    String id = _topology_context.getThisTaskId() + ":" + streamId;
    LOG.info(id + " groupType is " + groupType + ", outTasks is " + this.outTasks + ", localTasks" + localTasks);

}
 

/**
 * Gets information about who is consuming the outputs of the specified component, and how.
 *
 * @return Map from stream id to component id to the Grouping used.
 */
public Map<String, Map<String, Grouping>> getTargets(String componentId) {
    Map<String, Map<String, Grouping>> ret = new HashMap<>();
    for (String otherComponentId : getComponentIds()) {
        Map<GlobalStreamId, Grouping> inputs = getComponentCommon(otherComponentId).get_inputs();
        for (GlobalStreamId id : inputs.keySet()) {
            if (id.get_componentId().equals(componentId)) {
                Map<String, Grouping> curr = ret.get(id.get_streamId());
                if (curr == null)
                    curr = new HashMap<>();
                curr.put(otherComponentId, inputs.get(id));
                ret.put(id.get_streamId(), curr);
            }
        }
    }
    return ret;
}
 

protected void initCommon(String id, IComponent component, Number parallelism) throws IllegalArgumentException {
    ComponentCommon common = new ComponentCommon();
    common.set_inputs(new HashMap<GlobalStreamId, Grouping>());
    if (parallelism != null) {
        int dop = parallelism.intValue();
        if(dop < 1) {
            throw new IllegalArgumentException("Parallelism must be positive.");
        }
        common.set_parallelism_hint(dop);
    } else {
        common.set_parallelism_hint(1);
    }
    Map conf = component.getComponentConfiguration();
    if(conf!=null) common.set_json_conf(JSONValue.toJSONString(conf));
    _commons.put(id, common);
}
 

public void prepare(Map config, TopologyContext context, OutputCollector collector) {
    TimeCacheMap.ExpiredCallback<Object, TrackingInfo> callback = null;
    if (_delegate instanceof TimeoutCallback) {
        callback = new TimeoutItems();
    }
    _tracked = new TimeCacheMap<>(context.maxTopologyMessageTimeout(), callback);
    _collector = collector;
    _delegate.prepare(config, context, new OutputCollector(new CoordinatedOutputCollector(collector)));
    for (String component : Utils.get(context.getThisTargets(), Constants.COORDINATED_STREAM_ID, new HashMap<String, Grouping>()).keySet()) {
        for (Integer task : context.getComponentTasks(component)) {
            _countOutTasks.add(task);
        }
    }
    if (!_sourceArgs.isEmpty()) {
        _numSourceReports = 0;
        for (Entry<String, SourceArgs> entry : _sourceArgs.entrySet()) {
            if (entry.getValue().singleCount) {
                _numSourceReports += 1;
            } else {
                _numSourceReports += context.getComponentTasks(entry.getKey()).size();
            }
        }
    }
}
 

@SuppressWarnings("unchecked")
private static Map<String, Object> groupingToJSONableMap(Grouping grouping) {
    Map groupingMap = new HashMap<>();
    groupingMap.put("type", grouping.getSetField().toString());
    if (grouping.is_set_fields()) {
        groupingMap.put("fields", grouping.get_fields());
    }
    return groupingMap;
}
 

public Map<String, List<Integer>> getThisTargetComponentTasks() {
    Map<String, Map<String, Grouping>> outputGroupings = getThisTargets();
    Map<String, List<Integer>> ret = new HashMap<>();
    Set<String> targetComponents = new HashSet<>();

    for (Map.Entry<String, Map<String, Grouping>> entry : outputGroupings.entrySet()) {
        Map<String, Grouping> componentGrouping = entry.getValue();
        targetComponents.addAll(componentGrouping.keySet());
    }
    for (String component : targetComponents) {
        ret.put(component, getComponentTasks(component));
    }
    return ret;
}
 

/**
 * Add watermark stream to source components of window bolts
 */
private void maybeAddWatermarkInputs(ComponentCommon common, IRichBolt bolt) {
    if (bolt instanceof WindowedBoltExecutor) {
        Set<String> comps = new HashSet<>();
        for (GlobalStreamId globalStreamId : common.get_inputs().keySet()) {
            comps.add(globalStreamId.get_componentId());
        }

        for (String comp : comps) {
            common.put_to_inputs(
                    new GlobalStreamId(comp, Common.WATERMARK_STREAM_ID),
                    Grouping.all(new NullStruct()));
        }
    }
}
 

public Map<String, List<Integer>> getThisSourceComponentTasks() {
    Map<String, List<Integer>> ret = new HashMap<>();
    Map<GlobalStreamId, Grouping> sources = getThisSources();
    Set<String> sourceComponents = new HashSet<>();
    if (sources != null) {
        for (GlobalStreamId globalStreamId : sources.keySet()) {
            sourceComponents.add(globalStreamId.get_componentId());
        }
    }
    for (String component : sourceComponents) {
        ret.put(component, getComponentTasks(component));
    }
    return ret;
}
 

/**
 * For bolts that has incoming streams from spouts (the root bolts),
 * add checkpoint stream from checkpoint spout to its input. For other bolts,
 * add checkpoint stream from the previous bolt to its input.
 */
private void addCheckPointInputs(ComponentCommon component) {
    Set<GlobalStreamId> checkPointInputs = new HashSet<>();
    for (GlobalStreamId inputStream : component.get_inputs().keySet()) {
        String sourceId = inputStream.get_componentId();
        if (_spouts.containsKey(sourceId)) {
            checkPointInputs.add(new GlobalStreamId(CheckpointSpout.CHECKPOINT_COMPONENT_ID, CheckpointSpout.CHECKPOINT_STREAM_ID));
        } else {
            checkPointInputs.add(new GlobalStreamId(sourceId, CheckpointSpout.CHECKPOINT_STREAM_ID));
        }
    }
    for (GlobalStreamId streamId : checkPointInputs) {
        component.put_to_inputs(streamId, Grouping.all(new NullStruct()));
    }
}
 

private static boolean isIdentityPartition(PartitionNode n) {
    Grouping g = n.thriftGrouping;
    if(g.is_set_custom_serialized()) {
        CustomStreamGrouping csg = (CustomStreamGrouping) Utils.javaDeserialize(g.get_custom_serialized(), Serializable.class);
        return csg instanceof IdentityGrouping;
    }
    return false;
}
 

@Override
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
    _delegate.open(conf, context, new SpoutOutputCollector(new StreamOverrideCollector(collector)));
    _outputTasks = new ArrayList<>();
    for(String component: Utils.get(context.getThisTargets(),
                                    _coordStream,
                                    new HashMap<String, Grouping>()).keySet()) {
        _outputTasks.addAll(context.getComponentTasks(component));
    }
    _rand = new Random(Utils.secureRandomLong());
}
 

@Override
public void prepare(Map conf, TopologyContext context, OutputCollector collector) {
    _messageTimeoutMs = context.maxTopologyMessageTimeout() * 1000L;
    _lastRotate = System.currentTimeMillis();
    _batches = new RotatingMap<>(2);
    _context = context;
    _collector = collector;
    _coordCollector = new CoordinatedOutputCollector(new OutputCollector(collector));
    _coordOutputCollector = new BatchOutputCollectorImpl(new OutputCollector(_coordCollector));

    _coordConditions = (Map) context.getExecutorData("__coordConditions");
    if (_coordConditions == null) {
        _coordConditions = new HashMap<>();
        for (String batchGroup : _coordSpecs.keySet()) {
            CoordSpec spec = _coordSpecs.get(batchGroup);
            CoordCondition cond = new CoordCondition();
            cond.commitStream = spec.commitStream;
            cond.expectedTaskReports = 0;
            for (String comp : spec.coords.keySet()) {
                CoordType ct = spec.coords.get(comp);
                if (ct.equals(CoordType.single())) {
                    cond.expectedTaskReports += 1;
                } else {
                    cond.expectedTaskReports += context.getComponentTasks(comp).size();
                }
            }
            cond.targetTasks = new HashSet<>();
            for (String component : Utils.get(context.getThisTargets(),
                    COORD_STREAM(batchGroup),
                    new HashMap<String, Grouping>()).keySet()) {
                cond.targetTasks.addAll(context.getComponentTasks(component));
            }
            _coordConditions.put(batchGroup, cond);
        }
        context.setExecutorData("_coordConditions", _coordConditions);
    }
    _bolt.prepare(conf, context, _coordOutputCollector);
}
 

public BoltDeclarer globalGrouping(String componentId, String streamId) {
    return grouping(componentId, streamId, Grouping.fields(new ArrayList<String>()));
}
 

public BoltDeclarer fieldsGrouping(String componentId, String streamId, Fields fields) {
    return grouping(componentId, streamId, Grouping.fields(fields.toList()));
}
 

public BoltDeclarer shuffleGrouping(String componentId, String streamId) {
    return grouping(componentId, streamId, Grouping.shuffle(new NullStruct()));
}
 

public BoltDeclarer localOrShuffleGrouping(String componentId, String streamId) {
    return grouping(componentId, streamId, Grouping.local_or_shuffle(new NullStruct()));
}
 

@Override
public BoltDeclarer localFirstGrouping(String componentId, String streamId) {
    return grouping(componentId, streamId, Grouping.localFirst(new NullStruct()));
}
 

public BoltDeclarer noneGrouping(String componentId, String streamId) {
    return grouping(componentId, streamId, Grouping.none(new NullStruct()));
}
 

public BoltDeclarer allGrouping(String componentId, String streamId) {
    return grouping(componentId, streamId, Grouping.all(new NullStruct()));
}
 

public BoltDeclarer directGrouping(String componentId, String streamId) {
    return grouping(componentId, streamId, Grouping.direct(new NullStruct()));
}
 

@Override
public BoltDeclarer customGrouping(String componentId, String streamId, CustomStreamGrouping grouping) {
    return grouping(componentId, streamId, Grouping.custom_serialized(Utils.javaSerialize(grouping)));
}
 

public boolean initializeHbaseBolt(String name, String shuffleType) {

		try {

			String messageUpstreamComponent = dataComponents.get(dataComponents
					.size()-1);

			System.out.println("[OpenSOC] ------" + name
					+ " is initializing from " + messageUpstreamComponent);

			String tableName = config.getString("bolt.hbase.table.name")
					.toString();
			TupleTableConfig hbaseBoltConfig = new TupleTableConfig(tableName,
					config.getString("bolt.hbase.table.key.tuple.field.name")
							.toString(), config.getString(
							"bolt.hbase.table.timestamp.tuple.field.name")
							.toString());

			String allColumnFamiliesColumnQualifiers = config.getString(
					"bolt.hbase.table.fields").toString();
			// This is expected in the form
			// "<cf1>:<cq11>,<cq12>,<cq13>|<cf2>:<cq21>,<cq22>|......."
			String[] tokenizedColumnFamiliesWithColumnQualifiers = StringUtils
					.split(allColumnFamiliesColumnQualifiers, "\\|");
			for (String tokenizedColumnFamilyWithColumnQualifiers : tokenizedColumnFamiliesWithColumnQualifiers) {
				String[] cfCqTokens = StringUtils.split(
						tokenizedColumnFamilyWithColumnQualifiers, ":");
				String columnFamily = cfCqTokens[0];
				String[] columnQualifiers = StringUtils.split(cfCqTokens[1],
						",");
				for (String columnQualifier : columnQualifiers) {
					hbaseBoltConfig.addColumn(columnFamily, columnQualifier);
				}

				// hbaseBoltConfig.setDurability(Durability.valueOf(conf.get(
				// "storm.topology.pcap.bolt.hbase.durability").toString()));

				hbaseBoltConfig.setBatch(Boolean.valueOf(config.getString(
						"bolt.hbase.enable.batching").toString()));

				HBaseBolt hbase_bolt = new HBaseBolt(hbaseBoltConfig,
						config.getString("kafka.zk.list"),
						config.getString("kafka.zk.port"));
				hbase_bolt.setAutoAck(true);

				BoltDeclarer declarer = builder.setBolt(name, hbase_bolt,
						config.getInt("bolt.hbase.parallelism.hint"))
						.setNumTasks(config.getInt("bolt.hbase.num.tasks"));

				if (Grouping._Fields.CUSTOM_OBJECT.toString().equalsIgnoreCase(
						shuffleType)) {
					declarer.customGrouping(
							messageUpstreamComponent,
							"pcap_data_stream",
							new HBaseStreamPartitioner(
									hbaseBoltConfig.getTableName(),
									0,
									Integer.parseInt(conf
											.get("bolt.hbase.partitioner.region.info.refresh.interval.mins")
											.toString())));
				} else if (Grouping._Fields.DIRECT.toString().equalsIgnoreCase(
						shuffleType)) {
					declarer.fieldsGrouping(messageUpstreamComponent,
							"pcap_data_stream", new Fields("pcap_id"));
				}

			}
		} catch (Exception e) {
			e.printStackTrace();
			System.exit(0);
		}
		return true;
	}
 

private static PartitionNode makeIdentityPartition(Node basis) {
    return new PartitionNode(basis.streamId, basis.name, basis.allOutputFields,
        Grouping.custom_serialized(Utils.javaSerialize(new IdentityGrouping())));
}
 

private void readObject(ObjectInputStream ois) throws ClassNotFoundException, IOException {
    ois.defaultReadObject();
    byte[] ser = new byte[ois.readInt()];
    ois.readFully(ser);
    this.thriftGrouping = TridentUtils.thriftDeserialize(Grouping.class, ser);
}
 

public PartitionNode(String streamId, String name, Fields allOutputFields, Grouping grouping) {
    super(streamId, name, allOutputFields);
    this.thriftGrouping = grouping;
}
 

/**
 * ## Repartitioning Operation
 *
 * This method takes in a custom partitioning function that implements
 * {@link org.apache.storm.grouping.CustomStreamGrouping}
 *
 * @param grouping
 * @return
 */
public Stream partition(Grouping grouping) {
    if (_node instanceof PartitionNode) {
        return each(new Fields(), new TrueFilter()).partition(grouping);
    } else {
        return _topology.addSourcedNode(this, new PartitionNode(_node.streamId, _name, getOutputFields(), grouping));
    }
}