backtype.storm.utils.NimbusClient Java Examples

private static void completeTopology(String topologyName)
        throws Exception {
    Map conf = Utils.readStormConfig();
    NimbusClient client = NimbusClient.getConfiguredClient(conf);
    try {
        client.getClient().completeUpgrade(topologyName);
        CommandLineUtil.success("Successfully submit command complete_upgrade " + topologyName);
    } catch (Exception ex) {
        CommandLineUtil.error("Failed to perform complete_upgrade: " + ex.getMessage());
        ex.printStackTrace();
    } finally {
        if (client != null) {
            client.close();
        }
    }
}
 

public static void cleanCluster() {
    try {
        NimbusClient client = getNimbusClient(null);

        ClusterSummary clusterSummary = client.getClient().getClusterInfo();

        List<TopologySummary> topologySummaries = clusterSummary.get_topologies();

        KillOptions killOption = new KillOptions();
        killOption.set_wait_secs(1);
        for (TopologySummary topologySummary : topologySummaries) {
            client.getClient().killTopologyWithOpts(topologySummary.get_name(), killOption);
            LOG.info("Successfully kill " + topologySummary.get_name());
        }
    } catch (Exception e) {
        if (client != null) {
            client.close();
            client = null;

        }

        LOG.error("Failed to kill all topology ", e);
    }
}
 

public static void main(String[] args) {
    NimbusClient client = null;
    try {
        Map conf = Utils.readStormConfig();
        client = NimbusClient.getConfiguredClient(conf);

        if (args.length > 0 && !StringUtils.isBlank(args[0])) {
            String topologyName = args[0];
            TopologyInfo info = client.getClient().getTopologyInfoByName(topologyName);
            System.out.println("Successfully get topology info \n" + Utils.toPrettyJsonString(info));
        } else {
            ClusterSummary clusterSummary = client.getClient().getClusterInfo();
            System.out.println("Successfully get cluster info \n" + Utils.toPrettyJsonString(clusterSummary));
        }
    } catch (Exception e) {
        System.out.println(e.getMessage());
        e.printStackTrace();
        throw new RuntimeException(e);
    } finally {
        if (client != null) {
            client.close();
        }
    }
}
 

@RequestMapping("/supervisor/summary")
public Map supervisors(@PathVariable String clusterName){
    Map ret;
    NimbusClient client = null;
    try {
        client = NimbusClientManager.getNimbusClient(clusterName);
        ClusterSummary clusterSummary = client.getClient().getClusterInfo();
        ret = new HashMap<>();
        ret.put("supervisors", UIUtils.getSupervisorEntities(clusterSummary));
    } catch (Exception e) {
        NimbusClientManager.removeClient(clusterName);
        ret = UIUtils.exceptionJson(e);
        LOG.error(e.getMessage(), e);
    }
    return ret;
}
 

private static boolean topologyNameExists(Map conf, String name) {
    NimbusClient client = NimbusClient.getConfiguredClient(conf);
    try {
        ClusterSummary summary = client.getClient().getClusterInfo();
        for (TopologySummary s : summary.get_topologies()) {
            if (s.get_name().equals(name)) {
                return true;
            }
        }
        return false;

    } catch (Exception e) {
        throw new RuntimeException(e);
    } finally {
        client.close();
    }
}
 

@Override
public void stop(Application<StormEnvironment, StormTopology> executor, com.typesafe.config.Config config) {
    String appId = config.getString("appId");
    LOG.info("Stopping topology {} ...", appId);
    if (Objects.equals(config.getString("mode"), ApplicationEntity.Mode.CLUSTER.name())) {
        Nimbus.Client stormClient = NimbusClient.getConfiguredClient(getStormConfig(config)).getClient();
        try {
            stormClient.killTopologyWithOpts(appId, this.killOptions);
        } catch (NotAliveException | TException e) {
            LOG.error("Failed to kill topology named {}, due to: {}",appId,e.getMessage(),e.getCause());
            throw new RuntimeException(e.getMessage(),e);
        }
    } else {
        getLocalCluster().killTopologyWithOpts(appId, this.killOptions);
    }
    LOG.info("Stopped topology {}", appId);
}
 

public static void submitRebalance(String topologyName, RebalanceOptions options, Map conf) throws Exception {
    Map stormConf = Utils.readStormConfig();
    if (conf != null) {
        stormConf.putAll(conf);
    }

    NimbusClient client = null;
    try {
        client = NimbusClient.getConfiguredClient(stormConf);
        client.getClient().rebalance(topologyName, options);
    } catch (Exception e) {
        throw e;
    } finally {
        if (client != null) {
            client.close();
        }
    }
}
 

private static void rollbackTopology(String topologyName) {
    Map conf = Utils.readStormConfig();
    NimbusClient client = NimbusClient.getConfiguredClient(conf);
    try {
        // update jar
        client.getClient().rollbackTopology(topologyName);
        CommandLineUtil.success("Successfully submit command rollback_topology " + topologyName);
    } catch (Exception e) {
        e.printStackTrace();
        throw new RuntimeException(e);
    } finally {
        if (client != null) {
            client.close();
        }
    }
}
 

public static void main(String[] args) {
    if (args == null || args.length == 0) {
        throw new InvalidParameterException("Please input topology name");
    }

    String topologyName = args[0];
    NimbusClient client = null;
    try {
        Map conf = Utils.readStormConfig();
        client = NimbusClient.getConfiguredClient(conf);
        client.getClient().deactivate(topologyName);
        System.out.println("Successfully submit command deactivate " + topologyName);
    } catch (Exception e) {
        System.out.println(e.getMessage());
        e.printStackTrace();
        throw new RuntimeException(e);
    } finally {
        if (client != null) {
            client.close();
        }
    }
}
 

public static void main(String[] args) {
    if (args == null || args.length == 0) {
        throw new InvalidParameterException("Please input topology name");
    }

    String topologyName = args[0];

    NimbusClient client = null;
    try {

        Map conf = Utils.readStormConfig();
        client = NimbusClient.getConfiguredClient(conf);

        client.getClient().activate(topologyName);

        System.out.println("Successfully submit command activate " + topologyName);
    } catch (Exception e) {
        System.out.println(e.getMessage());
        e.printStackTrace();
        throw new RuntimeException(e);
    } finally {
        if (client != null) {
            client.close();
        }
    }
}
 

public static boolean isValidSupervisorHost(String clusterName, String host) {
    NimbusClient client = null;
    try {
        client = NimbusClientManager.getNimbusClient(clusterName);
        ClusterSummary clusterSummary = client.getClient().getClusterInfo();
        List<SupervisorEntity> supervisors = UIUtils.getSupervisorEntities(clusterSummary);
        for (SupervisorEntity s : supervisors) {
            if (s.getIp().equals(host) || s.getHost().equals(host)) {
                return true;
            }
        }
    } catch (Exception e) {
        e.printStackTrace();
    }
    return false;
}
 

@RequestMapping("/nimbus/summary")
public Map nimbus(@PathVariable String clusterName){
    Map ret;
    NimbusClient client = null;
    try {
        client = NimbusClientManager.getNimbusClient(clusterName);
        ClusterSummary clusterSummary = client.getClient().getClusterInfo();
        ret = new HashMap<>();
        ret.put("nimbus", UIUtils.getNimbusEntities(clusterSummary));
    } catch (Exception e) {
        NimbusClientManager.removeClient(clusterName);
        ret = UIUtils.exceptionJson(e);
        LOG.error(e.getMessage(), e);
    }
    return ret;
}
 

@RequestMapping("/topology/summary")
public Map topology(@PathVariable String clusterName){
    Map ret = new HashMap<>();
    NimbusClient client = null;
    try {
        client = NimbusClientManager.getNimbusClient(clusterName);
        ClusterSummary clusterSummary = client.getClient().getClusterInfo();
        List<TopologyEntity> topologies = UIUtils.getTopologyEntities(clusterSummary);
        ret.put("topologies", topologies);
    } catch (Exception e) {
        NimbusClientManager.removeClient(clusterName);
        ret = UIUtils.exceptionJson(e);
        LOG.error(e.getMessage(), e);
    }
    return ret;
}
 

public static NimbusClient getNimbusClient(String clusterName) throws Exception {
    Map conf = UIUtils.readUiConfig();
    NimbusClient client = clientManager.get(clusterName);
    if (client != null) {
        try {
            client.getClient().getVersion();
            LOG.info("get Nimbus Client from clientManager");
        } catch (Exception e) {
            LOG.info("Nimbus has been restarted, it begin to reconnect");
            client = null;
        }
    }

    if (client == null) {
        conf = UIUtils.resetZKConfig(conf, clusterName);
        client = NimbusClient.getConfiguredClient(conf);
        clientManager.put(clusterName, client);
    }

    return client;
}
 

@RequestMapping("/metrics")
public Map summaryMetrics(@PathVariable String clusterName, @PathVariable String topology) {
    int window = 60;    //we only get the 60s window
    Map<String, Object> ret = new HashMap<>();
    NimbusClient client = null;
    try {
        client = NimbusClientManager.getNimbusClient(clusterName);
        List<MetricInfo> infos = client.getClient().getMetrics(topology, MetaType.TOPOLOGY.getT());
        List<ChartSeries> metrics = UIUtils.getChartSeries(infos, window);
        ret.put("metrics", metrics);
    } catch (Exception e) {
        NimbusClientManager.removeClient(clusterName);
        ret = UIUtils.exceptionJson(e);
        LOG.error(e.getMessage(), e);
    }
    return ret;
}
 

public static Map<String, Object> getNimbusConf(String clusterName) {
    NimbusClient client = null;
    try {
        client = NimbusClientManager.getNimbusClient(clusterName);

        String jsonConf = client.getClient().getNimbusConf();
        Map<String, Object> nimbusConf =
                (Map<String, Object>) Utils.from_json(jsonConf);
        return nimbusConf;
    } catch (Exception e) {
        NimbusClientManager.removeClient(clusterName);
        LOG.error(e.getMessage(), e);
        return UIUtils.readUiConfig();
    }
}
 

public static boolean topologyNameExists(NimbusClient client, Map conf, String name) {
    try {
        client.getClient().getTopologyInfoByName(name);
        return true;
    } catch (Exception e) {
        return false;
    }
}
 

private static void flushClusterCache() {
    NimbusClient client = null;
    Map<String, ClusterEntity> clusterEntities = new HashMap<>();
    for (String name : UIUtils.clusterConfig.keySet()) {
        try {
            client = NimbusClientManager.getNimbusClient(name);
            clusterEntities.put(name, getClusterEntity(client.getClient().getClusterInfo(), name));
        } catch (Exception e) {
            NimbusClientManager.removeClient(name);
            LOG.error(e.getMessage(), e);
        }
    }
    clustersCache = clusterEntities;
}
 

public static void removeClient(String clusterName) {
    if (StringUtils.isBlank(clusterName)) {
        clusterName = UIDef.DEFAULT_CLUSTER_NAME;
    }
    NimbusClient client = (NimbusClient) clientManager.remove(clusterName);
    if (client != null) {
        client.close();
    }
}
 

public static String submitJar(Map conf, String localJar, String uploadLocation, NimbusClient client) {
    if (localJar == null) {
        throw new RuntimeException("Must submit topologies using the 'jstorm' client script so that " +
                "StormSubmitter knows which jar to upload.");
    }

    try {
        LOG.info("Uploading topology jar " + localJar + " to assigned location: " + uploadLocation);
        int bufferSize = 512 * 1024;
        Object maxBufSizeObject = conf.get(Config.NIMBUS_THRIFT_MAX_BUFFER_SIZE);
        if (maxBufSizeObject != null) {
            bufferSize = Utils.getInt(maxBufSizeObject) / 2;
        }

        BufferFileInputStream is = new BufferFileInputStream(localJar, bufferSize);
        while (true) {
            byte[] toSubmit = is.read();
            if (toSubmit.length == 0)
                break;
            client.getClient().uploadChunk(uploadLocation, ByteBuffer.wrap(toSubmit));
        }
        client.getClient().finishFileUpload(uploadLocation);
        LOG.info("Successfully uploaded topology jar to assigned location: " + uploadLocation);
        return uploadLocation;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}
 

public static Map<String, Object> getTopologyConf(String clusterName, String topologyId) {
    NimbusClient client = null;
    try {
        client = NimbusClientManager.getNimbusClient(clusterName);

        String jsonConf = client.getClient().getTopologyConf(topologyId);
        Map<String, Object> topologyConf =
                (Map<String, Object>) Utils.from_json(jsonConf);
        return topologyConf;
    } catch (Exception e) {
        NimbusClientManager.removeClient(clusterName);
        LOG.error(e.getMessage(), e);
        return getNimbusConf(clusterName);
    }
}
 

@Override
public void prepare(Map conf) {
    this.client = NimbusClient.getConfiguredClient(conf);
    if (conf != null) {
        this.bufferSize = JStormUtils.parseInt(conf.get(Config.STORM_BLOBSTORE_INPUTSTREAM_BUFFER_SIZE_BYTES), bufferSize);
    }
}
 

@Override
public boolean setClient(Map conf, NimbusClient client) {
    this.client = client;
    if (conf != null) {
        this.bufferSize = JStormUtils.parseInt(conf.get(Config.STORM_BLOBSTORE_INPUTSTREAM_BUFFER_SIZE_BYTES), bufferSize);
    }
    return true;
}
 

private static void upgradeTopology(String topologyName, String component, List<String> workers, int workerNum)
        throws Exception {
    Map conf = Utils.readStormConfig();
    NimbusClient client = NimbusClient.getConfiguredClient(conf);
    try {
        String topologyId = client.getClient().getTopologyId(topologyName);
        Map stormConf = (Map) Utils.from_json(client.getClient().getTopologyConf(topologyId));
        // check if TM is a separate worker
        TopologyInfo topologyInfo = client.getClient().getTopologyInfo(topologyId);
        for (TaskSummary taskSummary : topologyInfo.get_tasks()) {
            if (!taskSummary.get_status().equalsIgnoreCase("active")) {
                CommandLineUtil.error("Some of the tasks are not in ACTIVE state, cannot perform the upgrade!");
                return;
            }
        }

        if (!ConfigExtension.isTmSingleWorker(stormConf, topologyInfo.get_topology().get_numWorkers())) {
            CommandLineUtil.error("Gray upgrade requires that topology master to be a single worker, " +
                    "cannot perform the upgrade!");
            return;
        }

        client.getClient().grayUpgrade(topologyName, component, workers, workerNum);
        CommandLineUtil.success("Successfully submit command gray_upgrade " + topologyName);
    } catch (Exception ex) {
        CommandLineUtil.error("Failed to perform gray_upgrade: " + ex.getMessage());
        ex.printStackTrace();
    } finally {
        if (client != null) {
            client.close();
        }
    }
}
 

public static void main(String[] args) {
    if (args == null || args.length == 0) {
        throw new InvalidParameterException("Please input topology name");
    }

    String topologyName = args[0];
    NimbusClient client = null;
    try {
        Map conf = Utils.readStormConfig();
        client = NimbusClient.getConfiguredClient(conf);

        if (args.length == 1) {
            client.getClient().killTopology(topologyName);
        } else {
            int delaySeconds = Integer.parseInt(args[1]);
            KillOptions options = new KillOptions();
            options.set_wait_secs(delaySeconds);
            client.getClient().killTopologyWithOpts(topologyName, options);
        }

        System.out.println("Successfully submit command kill " + topologyName);
    } catch (Exception e) {
        System.out.println(e.getMessage());
        e.printStackTrace();
        throw new RuntimeException(e);
    } finally {
        if (client != null) {
            client.close();
        }
    }
}
 

public static void main(String[] args) {
    if (args == null || args.length < 2) {
        throw new InvalidParameterException("Please input action and hostname");
    }

    String action = args[0];
    String hostname = args[1];

    NimbusClient client = null;
    try {

        Map conf = Utils.readStormConfig();
        client = NimbusClient.getConfiguredClient(conf);

        if (action.equals("add"))
            client.getClient().setHostInBlackList(hostname);
        else {
            client.getClient().removeHostOutBlackList(hostname);
        }

        System.out.println("Successfully submit command blacklist with action:" + action + " and hostname :" + hostname);
    } catch (Exception e) {
        System.out.println(e.getMessage());
        e.printStackTrace();
        throw new RuntimeException(e);
    } finally {
        if (client != null) {
            client.close();
        }
    }
}
 

public static void main(String[] args) {
    if (args == null || args.length <= 1) {
        throw new InvalidParameterException("Please input topology name and enable flag");
    }

    String topologyName = args[0];
    NimbusClient client = null;
    try {
        Map conf = Utils.readStormConfig();
        client = NimbusClient.getConfiguredClient(conf);

        boolean isEnable = Boolean.valueOf(args[1]);

        MonitorOptions options = new MonitorOptions();
        options.set_isEnable(isEnable);

        client.getClient().metricMonitor(topologyName, options);

        String str = (isEnable) ? "enable" : "disable";
        System.out.println("Successfully submit command to " + str + " the monitor of " + topologyName);
    } catch (Exception e) {
        System.out.println(e.getMessage());
        e.printStackTrace();
        throw new RuntimeException(e);
    } finally {
        if (client != null) {
            client.close();
        }
    }
}
 

public static void main(String[] args) {
    if (args == null || args.length == 0) {
        throw new InvalidParameterException("Please input topology name!");
    }

    String topologyName = args[0];

    NimbusClient client = null;
    try {
        Map conf = Utils.readStormConfig();
        client = NimbusClient.getConfiguredClient(conf);

        System.out.println("It will take 15 ~ 100 seconds to restart, please wait patiently\n");

        if (args.length == 1) {
            client.getClient().restart(topologyName, null);
        } else {
            Map loadConf = Utils.loadConf(args[1]);
            String jsonConf = Utils.to_json(loadConf);
            System.out.println("New configuration:\n" + jsonConf);

            client.getClient().restart(topologyName, jsonConf);
        }

        System.out.println("Successfully submit command restart " + topologyName);
    } catch (Exception e) {
        System.out.println(e.getMessage());
        e.printStackTrace();
        throw new RuntimeException(e);
    } finally {
        if (client != null) {
            client.close();
        }
    }
}
 

public static boolean topologyNameExists(NimbusClient client, Map conf, String name) {
    try {
        client.getClient().getTopologyInfoByName(name);
        return true;
    } catch (Exception e) {
        return false;
    }
}
 

private static void submitJar(NimbusClient client, Map conf) {
    if (submittedJar == null) {
        try {
            LOG.info("Jar not uploaded to master yet. Submitting jar...");
            String localJar = System.getProperty("storm.jar");
            path = client.getClient().beginFileUpload();
            String[] pathCache = path.split("/");
            String uploadLocation = path + "/stormjar-" + pathCache[pathCache.length - 1] + ".jar";
            List<String> lib = (List<String>) conf.get(GenericOptionsParser.TOPOLOGY_LIB_NAME);
            Map<String, String> libPath = (Map<String, String>) conf.get(GenericOptionsParser.TOPOLOGY_LIB_PATH);
            if (lib != null && lib.size() != 0) {
                for (String libName : lib) {
                    String jarPath = path + "/lib/" + libName;
                    client.getClient().beginLibUpload(jarPath);
                    submitJar(conf, libPath.get(libName), jarPath, client);
                }

            } else {
                if (localJar == null) {
                    // no lib, no client jar
                    throw new RuntimeException("No client app jar found, please upload it");
                }
            }

            if (localJar != null) {
                submittedJar = submitJar(conf, localJar, uploadLocation, client);
            } else {
                // no client jar, but with lib jar
                client.getClient().finishFileUpload(uploadLocation);
            }
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    } else {
        LOG.info("Jar has already been uploaded to master. Will not submit again.");
    }
}