backtype.storm.generated.TopologySummary Java Examples

private static boolean topologyNameExists(Map conf, String name) {
    NimbusClient client = NimbusClient.getConfiguredClient(conf);
    try {
        ClusterSummary summary = client.getClient().getClusterInfo();
        for (TopologySummary s : summary.get_topologies()) {
            if (s.get_name().equals(name)) {
                return true;
            }
        }
        return false;

    } catch (Exception e) {
        throw new RuntimeException(e);
    } finally {
        client.close();
    }
}
 

public List<TopologyStatus> getTopologies() throws Exception {
    List<Topology> topologyDefinitions = dao.listTopologies();
    List<StreamingCluster> clusters = dao.listClusters();

    List<TopologyStatus> topologies = new ArrayList<>();
    for (Topology topologyDef : topologyDefinitions) {
        TopologySummary topologySummary = getTopologySummery(clusters, topologyDef);
        if (topologySummary != null) {
            TopologyStatus t = new TopologyStatus();
            t.setName(topologySummary.get_name());
            t.setId(topologySummary.get_id());
            t.setState(topologySummary.get_status());
            t.setTopology(topologyDef);
            topologies.add(t);
        }
    }
    return topologies;
}
 

@Test
public void testGetTopologies1() throws Exception {
    IMetadataDao dao = MetadataDaoFactory.getInstance().getMetadataDao();
    TopologyMgmtResourceImpl service = new TopologyMgmtResourceImpl();
    Field daoField = TopologyMgmtResourceImpl.class.getDeclaredField("dao");
    daoField.setAccessible(true);
    Field modifiersField = Field.class.getDeclaredField("modifiers");
    modifiersField.setAccessible(true);
    modifiersField.setInt(daoField, daoField.getModifiers() & ~Modifier.FINAL);
    daoField.set(null, dao);
    // set data
    Topology topology = new Topology("test", 1, 1);
    StreamingCluster cluster =new StreamingCluster();
    dao.clear();
    dao.addTopology(topology);
    dao.addCluster(cluster);
    TopologyMgmtResourceImpl spy = PowerMockito.spy(service);
    PowerMockito.doReturn(new TopologySummary()).when(spy,"getTopologySummery", Mockito.anyCollection(), Mockito.any(Topology.class));
    Assert.assertEquals(1, spy.getTopologies().size());
}
 

public static void cleanCluster() {
    try {
        NimbusClient client = getNimbusClient(null);

        ClusterSummary clusterSummary = client.getClient().getClusterInfo();

        List<TopologySummary> topologySummaries = clusterSummary.get_topologies();

        KillOptions killOption = new KillOptions();
        killOption.set_wait_secs(1);
        for (TopologySummary topologySummary : topologySummaries) {
            client.getClient().killTopologyWithOpts(topologySummary.get_name(), killOption);
            LOG.info("Successfully kill " + topologySummary.get_name());
        }
    } catch (Exception e) {
        if (client != null) {
            client.close();
            client = null;

        }

        LOG.error("Failed to kill all topology ", e);
    }
}
 

protected long getTopologyTPS(TopologySummary topology, Client client) throws NotAliveException, TException{
    long topologyTps = 0l;
    String topologyId = topology.get_id();
    if(topologyId.startsWith("ClusterMonitor")){
        return topologyTps;
    }
    TopologyInfo topologyInfo = client.getTopologyInfo(topologyId);
    if(topologyInfo == null){
        return topologyTps;
    }
    List<ExecutorSummary> executorSummaryList = topologyInfo.get_executors();
    for(ExecutorSummary executor : executorSummaryList){
        topologyTps += getComponentTPS(executor);
    }
    LOGGER.info("topology = " + topology.get_name() + ", tps = " + topologyTps);
    return topologyTps;
}
 

/**
 * @@@ Don't be compatible with Storm
 * 
 *     Here skip the logic
 * @param client
 * @param topology
 * @return
 * @throws Exception
 */
private HashSet<String> getComponents(Nimbus.Client client, String topology) throws Exception {
    HashSet<String> components = new HashSet<String>();
    ClusterSummary clusterSummary = client.getClusterInfo();
    TopologySummary topologySummary = null;
    for (TopologySummary ts : clusterSummary.get_topologies()) {
        if (topology.equals(ts.get_name())) {
            topologySummary = ts;
            break;
        }
    }
    if (topologySummary == null) {
        throw new IllegalArgumentException("topology: " + topology + " not found");
    } else {
        String id = topologySummary.get_id();
        // GetInfoOptions getInfoOpts = new GetInfoOptions();
        // getInfoOpts.set_num_err_choice(NumErrorsChoice.NONE);
        // TopologyInfo info = client.getTopologyInfoWithOpts(id, getInfoOpts);
        // for (ExecutorSummary es: info.get_executors()) {
        // components.add(es.get_component_id());
        // }
    }
    return components;
}
 

private void parseResults(List<TopologySummary> topologies) {
	if (topologies == null) {
		return;
	}

	for (TopologySummary topology : topologies) {
		String[] result = new String[RESULTSHEAD.length];
		result[0] = topology.get_name();
		result[1] = topology.get_status();
		result[2] = String.valueOf(topology.get_numWorkers());
		result[3] = String.valueOf(topology.get_uptimeSecs());
		results.add(result);
	}
}
 

@SuppressWarnings( {"rawtypes", "unused"})
private TopologySummary getTopologySummery(List<StreamingCluster> clusters, Topology topologyDef) throws Exception {
    Map stormConf = getStormConf(clusters, topologyDef.getClusterName());
    Nimbus.Client stormClient = NimbusClient.getConfiguredClient(stormConf).getClient();
    Optional<TopologySummary> tOp = stormClient.getClusterInfo().get_topologies().stream().filter(topology -> topology.get_name().equalsIgnoreCase(topologyDef.getName())).findFirst();
    if (tOp.isPresent()) {
        String id = tOp.get().get_id();
        //StormTopology stormTopology= stormClient.getTopology(id);
        return tOp.get();
    } else {
        return null;
    }
}
 

public static TopologySummary getTopologySummary(ClusterSummary cs, String name) {
  for (TopologySummary ts : cs.get_topologies()) {
    if (name.equals(ts.get_name())) {
      return ts;
    }
  }
  return null;
}
 

@Test
public void testGetTopologySummary() {
  ClusterSummary cs = mock(ClusterSummary.class);
  TopologySummary ts = mock(TopologySummary.class);
  String tsName = "benchmarks";
  String fakeName = "fake";

  when(cs.get_topologies()).thenReturn(Lists.newArrayList(ts));
  when(ts.get_name()).thenReturn(tsName);

  assertThat(MetricsUtils.getTopologySummary(cs, tsName)).isEqualTo(ts);
  assertThat(MetricsUtils.getTopologySummary(cs, fakeName)).isNull();
}
 

private void getClusterInfo(Client client) {
    try {
        ClusterSummary clusterSummary = client.getClusterInfo();
        List<SupervisorSummary> supervisorSummaryList = clusterSummary.get_supervisors();
        int totalWorkers = 0;
        int usedWorkers = 0;
        for(SupervisorSummary summary : supervisorSummaryList){
            totalWorkers += summary.get_num_workers() ;
            usedWorkers += summary.get_num_used_workers();
        }
        int freeWorkers = totalWorkers - usedWorkers;
        LOGGER.info("cluster totalWorkers = " + totalWorkers 
                + ", usedWorkers = " + usedWorkers 
                + ", freeWorkers  = " +  freeWorkers);
        
        HttpCatClient.sendMetric("ClusterMonitor", "freeSlots", "avg", String.valueOf(freeWorkers));
        HttpCatClient.sendMetric("ClusterMonitor", "totalSlots", "avg", String.valueOf(totalWorkers));
        
        List<TopologySummary> topologySummaryList = clusterSummary.get_topologies();
        long clusterTPS = 0l;
        for(TopologySummary topology : topologySummaryList){
            long topologyTPS = getTopologyTPS(topology, client);
            clusterTPS += topologyTPS;
            if(topology.get_name().startsWith("ClusterMonitor")){
                continue;
            }
            HttpCatClient.sendMetric(topology.get_name(), topology.get_name() + "-TPS", "avg", String.valueOf(topologyTPS));
        }
        HttpCatClient.sendMetric("ClusterMonitor", "ClusterEmitTPS", "avg", String.valueOf(clusterTPS));
        
    } catch (TException e) {
        initClient(configMap);
        LOGGER.error("get client info error.", e);
    }
    catch(NotAliveException nae){
        LOGGER.warn("topology is dead.", nae);
    }
}
 

public JStormApplicationResults(List<TopologySummary> topologies) {
	results = Lists.newArrayList();
	parseResults(topologies);
}
 

public static List<TopologySummary> getTopologySummary(StormClusterState stormClusterState,
                                                       Map<String, Assignment> assignments) throws Exception {
    List<TopologySummary> topologySummaries = new ArrayList<>();

    // get all active topology's StormBase
    Map<String, StormBase> bases = Cluster.get_all_StormBase(stormClusterState);
    for (Entry<String, StormBase> entry : bases.entrySet()) {

        String topologyId = entry.getKey();
        StormBase base = entry.getValue();

        Assignment assignment = stormClusterState.assignment_info(topologyId, null);
        if (assignment == null) {
            LOG.error("Failed to get assignment of topology: " + topologyId);
            continue;
        }
        assignments.put(topologyId, assignment);

        int num_workers = assignment.getWorkers().size();
        int num_tasks = getTopologyTaskNum(assignment);

        String errorString;
        if (Cluster.is_topology_exist_error(stormClusterState, topologyId)) {
            errorString = "Y";
        } else {
            errorString = "";
        }

        TopologySummary topology = new TopologySummary();
        topology.set_id(topologyId);
        topology.set_name(base.getStormName());
        topology.set_status(base.getStatusString());
        topology.set_uptimeSecs(TimeUtils.time_delta(base.getLanchTimeSecs()));
        topology.set_numWorkers(num_workers);
        topology.set_numTasks(num_tasks);
        topology.set_errorInfo(errorString);

        topologySummaries.add(topology);

    }

    return topologySummaries;
}
 

public void metrics(Nimbus.Client client, long now, MetricsState state) throws Exception {
    long totalStatted = 0;

    int componentParallelism = 0;
    boolean streamFound = false;
    ClusterSummary clusterSummary = client.getClusterInfo();
    TopologySummary topologySummary = null;
    for (TopologySummary ts : clusterSummary.get_topologies()) {
        if (_topology.equals(ts.get_name())) {
            topologySummary = ts;
            break;
        }
    }
    if (topologySummary == null) {
        throw new IllegalArgumentException("topology: " + _topology + " not found");
    } else {
        // String id = topologySummary.get_id();
        // GetInfoOptions getInfoOpts = new GetInfoOptions();
        // getInfoOpts.set_num_err_choice(NumErrorsChoice.NONE);
        // TopologyInfo info = client.getTopologyInfoWithOpts(id, getInfoOpts);
        // for (ExecutorSummary es: info.get_executors()) {
        // if (_component.equals(es.get_component_id())) {
        // componentParallelism ++;
        // ExecutorStats stats = es.get_stats();
        // if (stats != null) {
        // Map<String,Map<String,Long>> statted =
        // WATCH_EMITTED.equals(_watch) ? stats.get_emitted() : stats.get_transferred();
        // if ( statted != null) {
        // Map<String, Long> e2 = statted.get(":all-time");
        // if (e2 != null) {
        // Long stream = e2.get(_stream);
        // if (stream != null){
        // streamFound = true;
        // totalStatted += stream;
        // }
        // }
        // }
        // }
        // }
        // }
    }

    if (componentParallelism <= 0) {
        HashSet<String> components = getComponents(client, _topology);
        System.out.println("Available components for " + _topology + " :");
        System.out.println("------------------");
        for (String comp : components) {
            System.out.println(comp);
        }
        System.out.println("------------------");
        throw new IllegalArgumentException("component: " + _component + " not found");
    }

    if (!streamFound) {
        throw new IllegalArgumentException("stream: " + _stream + " not found");
    }
    long timeDelta = now - state.getLastTime();
    long stattedDelta = totalStatted - state.getLastStatted();
    state.setLastTime(now);
    state.setLastStatted(totalStatted);
    double throughput = (stattedDelta == 0 || timeDelta == 0) ? 0.0 : ((double) stattedDelta / (double) timeDelta);
    System.out.println(_topology + "\t" + _component + "\t" + componentParallelism + "\t" + _stream + "\t" + timeDelta + "\t" + stattedDelta + "\t"
            + throughput);
}