Java Code Examples for backtype.storm.utils.NimbusClient#close()

private static boolean topologyNameExists(Map conf, String name) {
    NimbusClient client = NimbusClient.getConfiguredClient(conf);
    try {
        ClusterSummary summary = client.getClient().getClusterInfo();
        for (TopologySummary s : summary.get_topologies()) {
            if (s.get_name().equals(name)) {
                return true;
            }
        }
        return false;

    } catch (Exception e) {
        throw new RuntimeException(e);
    } finally {
        client.close();
    }
}
 

public static void cleanCluster() {
    try {
        NimbusClient client = getNimbusClient(null);

        ClusterSummary clusterSummary = client.getClient().getClusterInfo();

        List<TopologySummary> topologySummaries = clusterSummary.get_topologies();

        KillOptions killOption = new KillOptions();
        killOption.set_wait_secs(1);
        for (TopologySummary topologySummary : topologySummaries) {
            client.getClient().killTopologyWithOpts(topologySummary.get_name(), killOption);
            LOG.info("Successfully kill " + topologySummary.get_name());
        }
    } catch (Exception e) {
        if (client != null) {
            client.close();
            client = null;

        }

        LOG.error("Failed to kill all topology ", e);
    }
}
 

public static void main(String[] args) {
    if (args == null || args.length == 0) {
        throw new InvalidParameterException("Please input topology name");
    }

    String topologyName = args[0];

    NimbusClient client = null;
    try {

        Map conf = Utils.readStormConfig();
        client = NimbusClient.getConfiguredClient(conf);

        client.getClient().activate(topologyName);

        System.out.println("Successfully submit command activate " + topologyName);
    } catch (Exception e) {
        System.out.println(e.getMessage());
        e.printStackTrace();
        throw new RuntimeException(e);
    } finally {
        if (client != null) {
            client.close();
        }
    }
}
 

private static void completeTopology(String topologyName)
        throws Exception {
    Map conf = Utils.readStormConfig();
    NimbusClient client = NimbusClient.getConfiguredClient(conf);
    try {
        client.getClient().completeUpgrade(topologyName);
        CommandLineUtil.success("Successfully submit command complete_upgrade " + topologyName);
    } catch (Exception ex) {
        CommandLineUtil.error("Failed to perform complete_upgrade: " + ex.getMessage());
        ex.printStackTrace();
    } finally {
        if (client != null) {
            client.close();
        }
    }
}
 

public static void main(String[] args) {
    NimbusClient client = null;
    try {
        Map conf = Utils.readStormConfig();
        client = NimbusClient.getConfiguredClient(conf);

        if (args.length > 0 && !StringUtils.isBlank(args[0])) {
            String topologyName = args[0];
            TopologyInfo info = client.getClient().getTopologyInfoByName(topologyName);
            System.out.println("Successfully get topology info \n" + Utils.toPrettyJsonString(info));
        } else {
            ClusterSummary clusterSummary = client.getClient().getClusterInfo();
            System.out.println("Successfully get cluster info \n" + Utils.toPrettyJsonString(clusterSummary));
        }
    } catch (Exception e) {
        System.out.println(e.getMessage());
        e.printStackTrace();
        throw new RuntimeException(e);
    } finally {
        if (client != null) {
            client.close();
        }
    }
}
 

public static void main(String[] args) {
    if (args == null || args.length == 0) {
        throw new InvalidParameterException("Please input topology name");
    }

    String topologyName = args[0];
    NimbusClient client = null;
    try {
        Map conf = Utils.readStormConfig();
        client = NimbusClient.getConfiguredClient(conf);
        client.getClient().deactivate(topologyName);
        System.out.println("Successfully submit command deactivate " + topologyName);
    } catch (Exception e) {
        System.out.println(e.getMessage());
        e.printStackTrace();
        throw new RuntimeException(e);
    } finally {
        if (client != null) {
            client.close();
        }
    }
}
 

public static void submitRebalance(String topologyName, RebalanceOptions options, Map conf) throws Exception {
    Map stormConf = Utils.readStormConfig();
    if (conf != null) {
        stormConf.putAll(conf);
    }

    NimbusClient client = null;
    try {
        client = NimbusClient.getConfiguredClient(stormConf);
        client.getClient().rebalance(topologyName, options);
    } catch (Exception e) {
        throw e;
    } finally {
        if (client != null) {
            client.close();
        }
    }
}
 

private static void rollbackTopology(String topologyName) {
    Map conf = Utils.readStormConfig();
    NimbusClient client = NimbusClient.getConfiguredClient(conf);
    try {
        // update jar
        client.getClient().rollbackTopology(topologyName);
        CommandLineUtil.success("Successfully submit command rollback_topology " + topologyName);
    } catch (Exception e) {
        e.printStackTrace();
        throw new RuntimeException(e);
    } finally {
        if (client != null) {
            client.close();
        }
    }
}
 

private static void upgradeTopology(String topologyName, String component, List<String> workers, int workerNum)
        throws Exception {
    Map conf = Utils.readStormConfig();
    NimbusClient client = NimbusClient.getConfiguredClient(conf);
    try {
        String topologyId = client.getClient().getTopologyId(topologyName);
        Map stormConf = (Map) Utils.from_json(client.getClient().getTopologyConf(topologyId));
        // check if TM is a separate worker
        TopologyInfo topologyInfo = client.getClient().getTopologyInfo(topologyId);
        for (TaskSummary taskSummary : topologyInfo.get_tasks()) {
            if (!taskSummary.get_status().equalsIgnoreCase("active")) {
                CommandLineUtil.error("Some of the tasks are not in ACTIVE state, cannot perform the upgrade!");
                return;
            }
        }

        if (!ConfigExtension.isTmSingleWorker(stormConf, topologyInfo.get_topology().get_numWorkers())) {
            CommandLineUtil.error("Gray upgrade requires that topology master to be a single worker, " +
                    "cannot perform the upgrade!");
            return;
        }

        client.getClient().grayUpgrade(topologyName, component, workers, workerNum);
        CommandLineUtil.success("Successfully submit command gray_upgrade " + topologyName);
    } catch (Exception ex) {
        CommandLineUtil.error("Failed to perform gray_upgrade: " + ex.getMessage());
        ex.printStackTrace();
    } finally {
        if (client != null) {
            client.close();
        }
    }
}
 

public static void main(String[] args) {
    if (args == null || args.length == 0) {
        throw new InvalidParameterException("Please input topology name!");
    }

    String topologyName = args[0];

    NimbusClient client = null;
    try {
        Map conf = Utils.readStormConfig();
        client = NimbusClient.getConfiguredClient(conf);

        System.out.println("It will take 15 ~ 100 seconds to restart, please wait patiently\n");

        if (args.length == 1) {
            client.getClient().restart(topologyName, null);
        } else {
            Map loadConf = Utils.loadConf(args[1]);
            String jsonConf = Utils.to_json(loadConf);
            System.out.println("New configuration:\n" + jsonConf);

            client.getClient().restart(topologyName, jsonConf);
        }

        System.out.println("Successfully submit command restart " + topologyName);
    } catch (Exception e) {
        System.out.println(e.getMessage());
        e.printStackTrace();
        throw new RuntimeException(e);
    } finally {
        if (client != null) {
            client.close();
        }
    }
}
 

public static void main(String[] args) {
    if (args == null || args.length <= 1) {
        throw new InvalidParameterException("Please input topology name and enable flag");
    }

    String topologyName = args[0];
    NimbusClient client = null;
    try {
        Map conf = Utils.readStormConfig();
        client = NimbusClient.getConfiguredClient(conf);

        boolean isEnable = Boolean.valueOf(args[1]);

        MonitorOptions options = new MonitorOptions();
        options.set_isEnable(isEnable);

        client.getClient().metricMonitor(topologyName, options);

        String str = (isEnable) ? "enable" : "disable";
        System.out.println("Successfully submit command to " + str + " the monitor of " + topologyName);
    } catch (Exception e) {
        System.out.println(e.getMessage());
        e.printStackTrace();
        throw new RuntimeException(e);
    } finally {
        if (client != null) {
            client.close();
        }
    }
}
 

public static void main(String[] args) {
    if (args == null || args.length < 2) {
        throw new InvalidParameterException("Please input action and hostname");
    }

    String action = args[0];
    String hostname = args[1];

    NimbusClient client = null;
    try {

        Map conf = Utils.readStormConfig();
        client = NimbusClient.getConfiguredClient(conf);

        if (action.equals("add"))
            client.getClient().setHostInBlackList(hostname);
        else {
            client.getClient().removeHostOutBlackList(hostname);
        }

        System.out.println("Successfully submit command blacklist with action:" + action + " and hostname :" + hostname);
    } catch (Exception e) {
        System.out.println(e.getMessage());
        e.printStackTrace();
        throw new RuntimeException(e);
    } finally {
        if (client != null) {
            client.close();
        }
    }
}
 

public static void main(String[] args) {
    if (args == null || args.length == 0) {
        throw new InvalidParameterException("Please input topology name");
    }

    String topologyName = args[0];
    NimbusClient client = null;
    try {
        Map conf = Utils.readStormConfig();
        client = NimbusClient.getConfiguredClient(conf);

        if (args.length == 1) {
            client.getClient().killTopology(topologyName);
        } else {
            int delaySeconds = Integer.parseInt(args[1]);
            KillOptions options = new KillOptions();
            options.set_wait_secs(delaySeconds);
            client.getClient().killTopologyWithOpts(topologyName, options);
        }

        System.out.println("Successfully submit command kill " + topologyName);
    } catch (Exception e) {
        System.out.println(e.getMessage());
        e.printStackTrace();
        throw new RuntimeException(e);
    } finally {
        if (client != null) {
            client.close();
        }
    }
}
 

public static void removeClient(String clusterName) {
    if (StringUtils.isBlank(clusterName)) {
        clusterName = UIDef.DEFAULT_CLUSTER_NAME;
    }
    NimbusClient client = (NimbusClient) clientManager.remove(clusterName);
    if (client != null) {
        client.close();
    }
}
 

public static List<String> getSupervisorHosts() throws Exception {
    try {
        List<String> hosts = new ArrayList<>();
        NimbusClient client = getNimbusClient(null);
        ClusterSummary clusterSummary = client.getClient().getClusterInfo();
        List<SupervisorSummary> supervisorSummaries = clusterSummary.get_supervisors();
        Collections.sort(supervisorSummaries, new Comparator<SupervisorSummary>() {

            @Override
            public int compare(SupervisorSummary o1, SupervisorSummary o2) {
                int o1Left = o1.get_numWorkers() - o1.get_numUsedWorkers();
                int o2Left = o2.get_numWorkers() - o2.get_numUsedWorkers();
                return o1Left - o2Left;
            }

        });

        for (SupervisorSummary supervisorSummary : supervisorSummaries) {
            hosts.add(supervisorSummary.get_host());
        }
        return hosts;
    } catch (Exception e) {
        if (client != null) {
            client.close();
            client = null;
        }

        LOG.error("Failed to kill all topologies ", e);
        throw new RuntimeException(e);
    }

}
 

/**
 * 任务提交
 * @param jstormTaskConfigDTO
 * @param submitTypeEnum
 * @return
 */
public static boolean taskSubmit(JstormTaskConfigDTO jstormTaskConfigDTO , TaskSubmitTypeEnum submitTypeEnum) {
    NimbusClient nimbusClient = null;
    boolean submitOk = false;
    try {
        jstormTaskConfigDTO.setSubmitType(submitTypeEnum.getValue());

        JstormYarnAppSubmitDTO  jstormYarnAppSubmitDTO = new JstormYarnAppSubmitDTO();

        TaskStartTimeLineDTO taskStartTimeLineDTO = new TaskStartTimeLineDTO(jstormTaskConfigDTO.getTaskName(),"准备提交任务，正在进行预处理");
        EngineBusiness.sysTaskStartTimeLine(taskStartTimeLineDTO);

        Integer processId = topologySubmit4CreateYarnApp(jstormTaskConfigDTO, taskStartTimeLineDTO, jstormYarnAppSubmitDTO);
        if (processId!=null) {
            nimbusClient = topologySubmit4ValidateCluster(jstormTaskConfigDTO, taskStartTimeLineDTO, jstormYarnAppSubmitDTO);
            if (nimbusClient != null) {
                submitOk = topologySubmit4SubmitTopology(jstormTaskConfigDTO, taskStartTimeLineDTO, jstormYarnAppSubmitDTO, nimbusClient,processId);
            }
        }
    } catch (Exception e) {
        LOGGER.error("topologySubmit is error",e);
    } finally {
        if(nimbusClient!=null){
            nimbusClient.close();
        }
    }
    return submitOk;
}
 

@Override
public void expire(String key, NimbusClient val) {
    LOGGER.info("Close connection of " + key);
    val.close();
}
 

@Override
public void expire(String key, NimbusClient val) {
    LOG.info("Close connection of " + key);
    val.close();
}
 

/**
 * Submit jar file
 * @param conf the topology-specific configuration. See {@link Config}.
 * @param localJar file path of the jar file to submit
 * @param listener progress listener to track the jar file upload
 * @return the remote location of the submitted jar
 */
public static String submitJar(Map conf, String localJar, ProgressListener listener) {
    if (localJar == null) {
        throw new RuntimeException("Must submit topologies using the 'storm' client script so that StormSubmitter knows which jar to upload.");
    }

    NimbusClient client = NimbusClient.getConfiguredClient(conf);
    try {
        String uploadLocation = client.getClient().beginFileUpload();
        LOG.info("Uploading topology jar " + localJar + " to assigned location: " + uploadLocation);
        BufferFileInputStream is = new BufferFileInputStream(localJar, THRIFT_CHUNK_SIZE_BYTES);

        long totalSize = new File(localJar).length();
        if (listener != null) {
            listener.onStart(localJar, uploadLocation, totalSize);
        }

        long bytesUploaded = 0;
        while (true) {
            byte[] toSubmit = is.read();
            bytesUploaded += toSubmit.length;
            if (listener != null) {
                listener.onProgress(localJar, uploadLocation, bytesUploaded, totalSize);
            }

            if (toSubmit.length == 0) {
                break;
            }
            client.getClient().uploadChunk(uploadLocation, ByteBuffer.wrap(toSubmit));
        }
        client.getClient().finishFileUpload(uploadLocation);

        if (listener != null) {
            listener.onCompleted(localJar, uploadLocation, totalSize);
        }

        LOG.info("Successfully uploaded topology jar to assigned location: " + uploadLocation);
        return uploadLocation;
    } catch (Exception e) {
        throw new RuntimeException(e);
    } finally {
        client.close();
    }
}
 

public Void call() throws Exception {
    NimbusClient nimbusClient = null;
    try{
        if(taskPO.getTaskStatus() != TaskStatusEnum.RUNNING.getValue()){
            return null;
        }

        JstormTaskConfigDTO taskConfigDTO = JSONObject.parseObject(jstormProcessPO.getTaskConfig(), JstormTaskConfigDTO.class);
        if(taskConfigDTO == null){
            return null;
        }

        String topId = jstormProcessPO.getTopId();
        taskMetricHbaseRecord = buildTaskMetricHbaseRecord(topId);

        TopologyInfo topologyInfo = null;
        nimbusClient = JStormClusterBusiness.getNimBusClientWithRetry(taskConfigDTO.getJstormZkHost(),taskConfigDTO.getJstormZkPort(),taskConfigDTO.getJstormZkRoot(),2,2);
        if(nimbusClient != null){
            topologyInfo = JStormClusterBusiness.getTopologyInfoWithRetry(nimbusClient,topId);
            if(topologyInfo == null){
                LOGGER.error("JstormMetricCollectExecutor 获取topologyInfo 失败! zkRoot=" + taskConfigDTO.getJstormZkRoot());
            }
        }else{
            LOGGER.error("JstormMetricCollectExecutor 获取nimbus client 失败! zkRoot=" + taskConfigDTO.getJstormZkRoot());
        }

        collectTopologyMetic(topologyInfo, topId);
        collectComponentMetic(topologyInfo, topId);
        collectWorkerMetic(topologyInfo, topId);
        processThreadStatus(topologyInfo, topId);

        if (isZkleader) {
            HBaseRecordUtils.send(taskMetricHbaseRecord);
        }
    }catch (Throwable e){
        LOGGER.error("JstormMetricCollectExecutor is error",e);
    }finally {
        if(nimbusClient!=null){
            nimbusClient.close();
        }
    }
    return null;
}