backtype.storm.utils.Utils#readStormConfig

Source File: TopologyMgmtResourceImpl.java From eagle with Apache License 2.0

6 votes

@SuppressWarnings( {"rawtypes", "unchecked"})
private Map getStormConf(List<StreamingCluster> clusters, String clusterId) throws Exception {
    Map<String, Object> stormConf = Utils.readStormConfig();
    if (clusterId == null) {
        stormConf.put(Config.NIMBUS_HOST, DEFAULT_NIMBUS_HOST);
        stormConf.put(Config.NIMBUS_THRIFT_PORT, DEFAULT_NIMBUS_THRIFT_PORT);
    } else {
        if (clusters == null) {
            clusters = dao.listClusters();
        }
        Optional<StreamingCluster> scOp = TopologyMgmtResourceHelper.findById(clusters, clusterId);
        StreamingCluster cluster;
        if (scOp.isPresent()) {
            cluster = scOp.get();
        } else {
            throw new Exception("Fail to find cluster: " + clusterId);
        }
        stormConf.put(Config.NIMBUS_HOST, cluster.getDeployments().getOrDefault(StreamingCluster.NIMBUS_HOST, DEFAULT_NIMBUS_HOST));
        stormConf.put(Config.NIMBUS_THRIFT_PORT, Integer.valueOf(cluster.getDeployments().get(StreamingCluster.NIMBUS_THRIFT_PORT)));
    }
    return stormConf;
}

Source File: rollback_topology.java From jstorm with Apache License 2.0

6 votes

private static void rollbackTopology(String topologyName) {
    Map conf = Utils.readStormConfig();
    NimbusClient client = NimbusClient.getConfiguredClient(conf);
    try {
        // update jar
        client.getClient().rollbackTopology(topologyName);
        CommandLineUtil.success("Successfully submit command rollback_topology " + topologyName);
    } catch (Exception e) {
        e.printStackTrace();
        throw new RuntimeException(e);
    } finally {
        if (client != null) {
            client.close();
        }
    }
}

Source File: HealthCheck.java From jstorm with Apache License 2.0

6 votes

public static HealthStatus check() {
    Map conf = Utils.readStormConfig();

    long timeout = ConfigExtension.getStormHealthTimeoutMs(conf);
    // 1. check panic dir
    String panicPath = ConfigExtension.getStormMachineResourcePanicCheckDir(conf);
    if (!isHealthyUnderPath(panicPath, timeout)) {
        return HealthStatus.PANIC;
    }

    // 2. check error dir
    String errorPath = ConfigExtension.getStormMachineResourceErrorCheckDir(conf);
    if (!isHealthyUnderPath(errorPath, timeout)) {
        return HealthStatus.ERROR;
    }

    // 3. check warn dir
    String warnPath = ConfigExtension.getStormMachineResourceWarningCheckDir(conf);
    if (!isHealthyUnderPath(warnPath, timeout)) {
        return HealthStatus.WARN;
    }

    return HealthStatus.INFO;
}

Source File: activate.java From jstorm with Apache License 2.0

6 votes

public static void main(String[] args) {
    if (args == null || args.length == 0) {
        throw new InvalidParameterException("Please input topology name");
    }

    String topologyName = args[0];

    NimbusClient client = null;
    try {

        Map conf = Utils.readStormConfig();
        client = NimbusClient.getConfiguredClient(conf);

        client.getClient().activate(topologyName);

        System.out.println("Successfully submit command activate " + topologyName);
    } catch (Exception e) {
        System.out.println(e.getMessage());
        e.printStackTrace();
        throw new RuntimeException(e);
    } finally {
        if (client != null) {
            client.close();
        }
    }
}

Source File: LocalUtils.java From jstorm with Apache License 2.0

6 votes

@SuppressWarnings("unchecked")
public static Map getLocalConf(int port) {
    Map conf = Utils.readStormConfig();
    conf.putAll(getLocalBaseConf());

    List<String> zkServers = new ArrayList<>(1);
    zkServers.add("localhost");

    conf.put(Config.STORM_ZOOKEEPER_SERVERS, zkServers);
    conf.put(Config.STORM_ZOOKEEPER_PORT, port);

    ConfigExtension.setTopologyDebugRecvTuple(conf, true);
    conf.put(Config.TOPOLOGY_DEBUG, true);

    conf.put(ConfigExtension.TOPOLOGY_BACKPRESSURE_ENABLE, false);

    return conf;
}

Source File: ZkTool.java From jstorm with Apache License 2.0

6 votes

public static void main(String[] args) throws Exception {
    if (args.length < 2) {
        System.out.println("Invalid parameter");
        usage();
        return;
    }

    conf = Utils.readStormConfig();

    if (args[0].equalsIgnoreCase(READ_CMD)) {

        readData(args[1]);

    } else if (args[0].equalsIgnoreCase(RM_CMD)) {
        rmBakTopology(args[1]);
    } else if (args[0].equalsIgnoreCase(LIST_CMD)) {
        list(args[1]);
    } else if (args[0].equalsIgnoreCase(CLEAN_CMD)) {
        cleanTopology(args[1]);
    }

}

Source File: deactivate.java From jstorm with Apache License 2.0

6 votes

public static void main(String[] args) {
    if (args == null || args.length == 0) {
        throw new InvalidParameterException("Please input topology name");
    }

    String topologyName = args[0];
    NimbusClient client = null;
    try {
        Map conf = Utils.readStormConfig();
        client = NimbusClient.getConfiguredClient(conf);
        client.getClient().deactivate(topologyName);
        System.out.println("Successfully submit command deactivate " + topologyName);
    } catch (Exception e) {
        System.out.println(e.getMessage());
        e.printStackTrace();
        throw new RuntimeException(e);
    } finally {
        if (client != null) {
            client.close();
        }
    }
}

Source File: ZooKeeperDataViewTest.java From jstorm with Apache License 2.0

6 votes

@BeforeClass
public static void init() {
    String CONFIG_PATH = System.getProperty("user.home") + CONFIG_FILE;
    File file = new File(CONFIG_PATH);
    if (file.exists() == false) {
        SKIP = true;
        return;
    }

    try {
        zkobj = new Zookeeper();
        System.getProperties().setProperty("storm.conf.file", CONFIG_PATH);
        Map conf = Utils.readStormConfig();
        zk = zkobj.mkClient(conf,
                (List<String>) conf.get(Config.STORM_ZOOKEEPER_SERVERS),
                conf.get(Config.STORM_ZOOKEEPER_PORT),
                (String) conf.get(Config.STORM_ZOOKEEPER_ROOT));
        gson = new GsonBuilder().setPrettyPrinting().create();
    }catch(Throwable e) {
        e.printStackTrace();
        SKIP = true;
    }
}

Source File: complete_upgrade.java From jstorm with Apache License 2.0

6 votes

private static void completeTopology(String topologyName)
        throws Exception {
    Map conf = Utils.readStormConfig();
    NimbusClient client = NimbusClient.getConfiguredClient(conf);
    try {
        client.getClient().completeUpgrade(topologyName);
        CommandLineUtil.success("Successfully submit command complete_upgrade " + topologyName);
    } catch (Exception ex) {
        CommandLineUtil.error("Failed to perform complete_upgrade: " + ex.getMessage());
        ex.printStackTrace();
    } finally {
        if (client != null) {
            client.close();
        }
    }
}

Source File: TestReachTopology.java From jstorm with Apache License 2.0

5 votes

/**
 * @param args
 * @throws DRPCExecutionException
 * @throws TException
 */
public static void main(String[] args) throws Exception {
    
    if (args.length < 1) {
        throw new IllegalArgumentException("Invalid parameter");
    }
    Map conf = Utils.readStormConfig();
    // "foo.com/blog/1" "engineering.twitter.com/blog/5"
    DRPCClient client = new DRPCClient(conf, args[0], 4772);
    String result = client.execute(ReachTopology.TOPOLOGY_NAME, "tech.backtype.com/blog/123");
    
    System.out.println("\n!!! Drpc result:" + result);
}

Source File: JStormHelper.java From jstorm with Apache License 2.0

5 votes

@Override
public <T> Object execute(T... args) {
    String topologyName = (String) args[0];

    try {
        checkError(conf, topologyName);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }

    Map stormConf = Utils.readStormConfig();
    stormConf.putAll(conf);
    Map<String, Double> ret = JStormUtils.getMetrics(stormConf, topologyName, MetaType.TASK, null);
    for (Entry<String, Double> entry : ret.entrySet()) {
        String key = entry.getKey();
        Double value = entry.getValue();

        if (key.indexOf(MetricDef.FAILED_NUM) > 0) {
            Assert.assertTrue(key + " fail number should == 0", value == 0);
        } else if (key.indexOf(MetricDef.ACKED_NUM) > 0) {
            if (value == 0.0) {
                LOG.warn(key + ":" + value);
            } else {
                LOG.info(key + ":" + value);
            }

        } else if (key.indexOf(MetricDef.EMMITTED_NUM) > 0) {
            if (value == 0.0) {
                LOG.warn(key + ":" + value);
            } else {
                LOG.info(key + ":" + value);
            }
        }
    }

    return ret;
}

Source File: ProcessLauncher.java From jstorm with Apache License 2.0

5 votes

public static int getSleepSeconds() {
    Map<Object, Object> conf;
    try {
        conf = Utils.readStormConfig();
    } catch (Exception e) {
        conf = new HashMap<>();
    }
    return ConfigExtension.getProcessLauncherSleepSeconds(conf);
}

Source File: NimbusServer.java From jstorm with Apache License 2.0

5 votes

public static void main(String[] args) throws Exception {
    Thread.setDefaultUncaughtExceptionHandler(new DefaultUncaughtExceptionHandler());
    // read configuration files
    @SuppressWarnings("rawtypes")
    Map config = Utils.readStormConfig();
    JStormServerUtils.startTaobaoJvmMonitor();
    NimbusServer instance = new NimbusServer();
    INimbus iNimbus = new DefaultInimbus();
    instance.launchServer(config, iNimbus);
}

Source File: SandBoxMaker.java From jstorm with Apache License 2.0

5 votes

public static void main(String[] args) {
    Map<Object, Object> conf = Utils.readStormConfig();
    conf.put("java.sandbox.enable", true);
    SandBoxMaker maker = new SandBoxMaker(conf);
    try {
        System.out.println("sandboxPolicy:" + maker.sandboxPolicy("simple", new HashMap<String, String>()));
    } catch (IOException e) {
        e.printStackTrace();
    }
}

Source File: Supervisor.java From jstorm with Apache License 2.0

5 votes

/**
 * start supervisor
 */
public void run() {
    SupervisorManger supervisorManager;
    try {
        Map<Object, Object> conf = Utils.readStormConfig();

        StormConfig.validate_distributed_mode(conf);

        createPid(conf);

        supervisorManager = mkSupervisor(conf, null);

        JStormUtils.redirectOutput("/dev/null");

        initShutdownHook(supervisorManager);

        while (!supervisorManager.isFinishShutdown()) {
            try {
                Thread.sleep(1000);
            } catch (InterruptedException ignored) {
            }
        }

    } catch (Throwable e) {
        if (e instanceof OutOfMemoryError) {
            LOG.error("Halting due to out of memory error...");
        }
        LOG.error("Fail to run supervisor ", e);
        System.exit(1);
    } finally {
        LOG.info("Shutdown supervisor!!!");
    }

}

Source File: blacklist.java From jstorm with Apache License 2.0

5 votes

public static void main(String[] args) {
    if (args == null || args.length < 2) {
        throw new InvalidParameterException("Please input action and hostname");
    }

    String action = args[0];
    String hostname = args[1];

    NimbusClient client = null;
    try {

        Map conf = Utils.readStormConfig();
        client = NimbusClient.getConfiguredClient(conf);

        if (action.equals("add"))
            client.getClient().setHostInBlackList(hostname);
        else {
            client.getClient().removeHostOutBlackList(hostname);
        }

        System.out.println("Successfully submit command blacklist with action:" + action + " and hostname :" + hostname);
    } catch (Exception e) {
        System.out.println(e.getMessage());
        e.printStackTrace();
        throw new RuntimeException(e);
    } finally {
        if (client != null) {
            client.close();
        }
    }
}

Source File: BlobStoreTest.java From jstorm with Apache License 2.0

5 votes

private LocalFsBlobStore initLocalFs() {
    LocalFsBlobStore store = new LocalFsBlobStore();
    // Spy object that tries to mock the real object store
    LocalFsBlobStore spy = spy(store);
    Mockito.doNothing().when(spy).checkForBlobUpdate("test");
    Mockito.doNothing().when(spy).checkForBlobUpdate("other");
    Mockito.doNothing().when(spy).checkForBlobUpdate("test-empty-subject-WE");
    Mockito.doNothing().when(spy).checkForBlobUpdate("test-empty-subject-DEF");
    Mockito.doNothing().when(spy).checkForBlobUpdate("test-empty-acls");
    Map conf = Utils.readStormConfig();
    conf.put(Config.STORM_LOCAL_DIR, baseFile.getAbsolutePath());
    conf.put(Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN,"org.apache.storm.security.auth.DefaultPrincipalToLocal");
    spy.prepare(conf, null, null);
    return spy;
}

Source File: config_value.java From jstorm with Apache License 2.0

5 votes

public static void main(String[] args) {
    if (args == null || args.length == 0) {
        throw new InvalidParameterException("Please input key name");
    }

    String key = args[0];
    Map conf = Utils.readStormConfig();
    System.out.print("VALUE: " + String.valueOf(conf.get(key)));
}

Source File: UserDefinedWorkerTopology.java From jstorm with Apache License 2.0

4 votes

public void verifyAssignment(String topologyName) {

            
            Set<ResourceWorkerSlot> spoutResourceWorkerSlots = new HashSet<>();
            Set<ResourceWorkerSlot> bolt1ResourceWorkerSlots = new HashSet<>();
            Set<ResourceWorkerSlot> bolt2ResourceWorkerSlots = new HashSet<>();
            
            NimbusClientWrapper client = new NimbusClientWrapper();
            try {
                Map nimbusConf = Utils.readStormConfig();
                client.init(nimbusConf);
                
                String topologyId = client.getClient().getTopologyId(topologyName);
                
                TopologyInfo topologyInfo = client.getClient().getTopologyInfo(topologyId);
                
                Assignment assignment = JStormHelper.getAssignment(topologyId, conf);
                Set<ResourceWorkerSlot> workerSet = assignment.getWorkers();
                
                List<ComponentSummary>  componentList = topologyInfo.get_components();
                for (ComponentSummary component : componentList) {
                    if (SPOUT_NAME.equals(component.get_name())) {
                        spoutResourceWorkerSlots = getComponentWorkers(component, workerSet);
                    }else if (BOLT1_NAME.equals(component.get_name())) {
                        bolt1ResourceWorkerSlots = getComponentWorkers(component, workerSet);
                    }else if (BOLT2_NAME.equals(component.get_name())) {
                        bolt2ResourceWorkerSlots = getComponentWorkers(component, workerSet);
                    }
                }
                
            }catch(Exception e) {
                Assert.fail("Fail to get workerSlots");
            }finally {
                client.cleanup();
            }
            
            
            verifySpoutAssignment(spoutResourceWorkerSlots);
            verifyBolt1Assignment(spoutResourceWorkerSlots, bolt1ResourceWorkerSlots);
            verifyBolt2Assignment(bolt1ResourceWorkerSlots, bolt2ResourceWorkerSlots);
            
        }

Source File: StormConfig.java From jstorm with Apache License 2.0

4 votes

public static Map read_storm_config() {
    return Utils.readStormConfig();
}

Java Code Examples for backtype.storm.utils.Utils#readStormConfig()