org.apache.hadoop.registry.client.types.ServiceRecord#set

Source File: Executor.java From jstorm with Apache License 2.0

6 votes

public boolean needUpgrade() {
    String containerPath = RegistryUtils.componentPath(
            JOYConstants.APP_TYPE, this.executorMeta.getInstanceName(),
            this.executorMeta.getApplicationId(), this.executorMeta.getRunningContainer());

    try {
        if (registryOperations.exists(containerPath)) {
            ServiceRecord sr = registryOperations.resolve(containerPath);
            if (sr.get(JOYConstants.NEED_UPGRADE) != null && sr.get(JOYConstants.NEED_UPGRADE).equals(JOYConstants.TRUE)) {
                sr.set(JOYConstants.NEED_UPGRADE, JOYConstants.FALSE);
                registryOperations.bind(containerPath, sr, BindFlags.OVERWRITE);
                LOG.info(JOYConstants.NEED_UPGRADE);
                return true;
            }
        }
    } catch (IOException e) {
        e.printStackTrace();
    }
    return false;
}

Source File: JStormOnYarnBusiness.java From PoseidonX with Apache License 2.0

6 votes

/**
 * 删除应用
 * @param zkHosts
 * @param zkPort
 * @param appId
 */
public static void killApplication(List<String> zkHosts, Integer zkPort, String appId, String taskName) throws Exception{
    if(StringUtils.isNotBlank(appId) && StringUtils.isNotBlank(taskName)){

        String applicationPath = YarnZkRegistryBusiness.PathBuilder.applicationPath(taskName,appId);
        ServiceRecord serviceRecord = YarnZkRegistryBusiness.resolve(applicationPath);
        if(serviceRecord !=null){
            serviceRecord.set("killed","true");
            YarnZkRegistryBusiness.bind(applicationPath,serviceRecord);
        }
        ApplicationReport applicationReport = YarnClientProxy.getApplicationReportByAppId(appId);
        if(applicationReport !=null){
            try{
                YarnClientProxy.killApplicationByAppId(appId);
            }catch (Exception e){
                if(serviceRecord != null){
                    serviceRecord.set("killed","false");
                    YarnZkRegistryBusiness.bind(applicationPath,serviceRecord);
                }
                throw e;
            }
        }
    }
    JStormOnYarnBusiness.clearZkInfo(zkHosts,zkPort,taskName);
}

Source File: RegistryTestHelper.java From hadoop with Apache License 2.0

5 votes

public static ServiceRecord createRecord(String id, String persistence,
    String description) {
  ServiceRecord serviceRecord = new ServiceRecord();
  serviceRecord.set(YarnRegistryAttributes.YARN_ID, id);
  serviceRecord.description = description;
  serviceRecord.set(YarnRegistryAttributes.YARN_PERSISTENCE, persistence);
  return serviceRecord;
}

Source File: RegistryTestHelper.java From big-c with Apache License 2.0

5 votes

public static ServiceRecord createRecord(String id, String persistence,
    String description) {
  ServiceRecord serviceRecord = new ServiceRecord();
  serviceRecord.set(YarnRegistryAttributes.YARN_ID, id);
  serviceRecord.description = description;
  serviceRecord.set(YarnRegistryAttributes.YARN_PERSISTENCE, persistence);
  return serviceRecord;
}

Source File: RegistryTestHelper.java From big-c with Apache License 2.0

5 votes

/**
 * Create a service entry with the sample endpoints
 * @param persistence persistence policy
 * @return the record
 * @throws IOException on a failure
 */
public static ServiceRecord buildExampleServiceEntry(String persistence) throws
    IOException,
    URISyntaxException {
  ServiceRecord record = new ServiceRecord();
  record.set(YarnRegistryAttributes.YARN_ID, "example-0001");
  record.set(YarnRegistryAttributes.YARN_PERSISTENCE, persistence);
  addSampleEndpoints(record, "namenode");
  return record;
}

Source File: JstormMaster.java From jstorm with Apache License 2.0

5 votes

private ServiceRecord setupServiceRecord() {
    ServiceRecord application = new ServiceRecord();
    application.set(YarnRegistryAttributes.YARN_ID, jstormMasterContext.appAttemptID.getApplicationId().toString());
    application.description = JOYConstants.AM;
    application.set(YarnRegistryAttributes.YARN_PERSISTENCE,
            PersistencePolicies.PERMANENT);
    Map<String, String> addresses = new HashMap<String, String>();
    addresses.put(JOYConstants.HOST, jstormMasterContext.appMasterHostname);
    addresses.put(JOYConstants.PORT, String.valueOf(jstormMasterContext.appMasterThriftPort));
    Endpoint endpoint = new Endpoint(JOYConstants.HTTP, JOYConstants.HOST_PORT, JOYConstants.RPC, addresses);
    application.addExternalEndpoint(endpoint);
    return application;
}

Source File: TestMarshalling.java From big-c with Apache License 2.0

5 votes

@Test
public void testUnknownFieldsRoundTrip() throws Throwable {
  ServiceRecord record =
      createRecord(PersistencePolicies.APPLICATION_ATTEMPT);
  record.set("key", "value");
  record.set("intval", "2");
  assertEquals("value", record.get("key"));
  assertEquals("2", record.get("intval"));
  assertNull(record.get("null"));
  assertEquals("defval", record.get("null", "defval"));
  byte[] bytes = marshal.toBytes(record);
  ServiceRecord r2 = marshal.fromBytes("", bytes);
  assertEquals("value", r2.get("key"));
  assertEquals("2", r2.get("intval"));
}

Source File: TestMarshalling.java From big-c with Apache License 2.0

5 votes

@Test
public void testRoundTrip() throws Throwable {
  String persistence = PersistencePolicies.PERMANENT;
  ServiceRecord record = createRecord(persistence);
  record.set("customkey", "customvalue");
  record.set("customkey2", "customvalue2");
  RegistryTypeUtils.validateServiceRecord("", record);
  LOG.info(marshal.toJson(record));
  byte[] bytes = marshal.toBytes(record);
  ServiceRecord r2 = marshal.fromBytes("", bytes);
  assertMatches(record, r2);
  RegistryTypeUtils.validateServiceRecord("", r2);
}

Source File: TestRegistryOperations.java From big-c with Apache License 2.0

5 votes

@Test(expected = PathNotFoundException.class)
public void testPutNoParent2() throws Throwable {
  ServiceRecord record = new ServiceRecord();
  record.set(YarnRegistryAttributes.YARN_ID, "testPutNoParent");
  String path = "/path/without/parent";
  operations.bind(path, record, 0);
}

Source File: TestRegistryOperations.java From big-c with Apache License 2.0

5 votes

@Test
public void testPutNoParent() throws Throwable {
  ServiceRecord record = new ServiceRecord();
  record.set(YarnRegistryAttributes.YARN_ID, "testPutNoParent");
  String path = "/path/without/parent";
  try {
    operations.bind(path, record, 0);
    // didn't get a failure
    // trouble
    RegistryPathStatus stat = operations.stat(path);
    fail("Got a status " + stat);
  } catch (PathNotFoundException expected) {
    // expected
  }
}

Source File: RegistryTestHelper.java From hadoop with Apache License 2.0

5 votes

/**
 * Create a service entry with the sample endpoints
 * @param persistence persistence policy
 * @return the record
 * @throws IOException on a failure
 */
public static ServiceRecord buildExampleServiceEntry(String persistence) throws
    IOException,
    URISyntaxException {
  ServiceRecord record = new ServiceRecord();
  record.set(YarnRegistryAttributes.YARN_ID, "example-0001");
  record.set(YarnRegistryAttributes.YARN_PERSISTENCE, persistence);
  addSampleEndpoints(record, "namenode");
  return record;
}

Source File: SlotPortsView.java From jstorm with Apache License 2.0

5 votes

/**
 * see if anyone is updating host's port list, if not start , update this host itself
 * timeout is 45 seconds
 *
 * @param hostPath
 * @throws InterruptedException
 * @throws IOException
 */
private void tryHostLock(String hostPath) throws Exception {

    //if path has created 60 seconds ago, then delete
    if (registryOperations.exists(hostPath)) {
        try {
            ServiceRecord host = registryOperations.resolve(hostPath);
            Long cTime = Long.parseLong(host.get(JOYConstants.CTIME, JOYConstants.DEFAULT_CTIME));
            Date now = new Date();
            if (now.getTime() - cTime > JOYConstants.HOST_LOCK_TIMEOUT || cTime > now.getTime())
                registryOperations.delete(hostPath, true);
        } catch (Exception ex) {
            LOG.error(ex);
        }
    }

    int failedCount = JOYConstants.RETRY_TIMES;
    while (!registryOperations.mknode(hostPath, true)) {
        Thread.sleep(JOYConstants.SLEEP_INTERVAL);
        failedCount--;
        if (failedCount <= 0)
            break;
    }

    if (failedCount > 0) {
        ServiceRecord sr = new ServiceRecord();
        Date date = new Date();
        date.getTime();
        sr.set(JOYConstants.CTIME, String.valueOf(date.getTime()));
        registryOperations.bind(hostPath, sr, BindFlags.OVERWRITE);
        return;
    }
    throw new Exception("can't get host lock");
}

Source File: TestMarshalling.java From hadoop with Apache License 2.0

5 votes

@Test
public void testUnknownFieldsRoundTrip() throws Throwable {
  ServiceRecord record =
      createRecord(PersistencePolicies.APPLICATION_ATTEMPT);
  record.set("key", "value");
  record.set("intval", "2");
  assertEquals("value", record.get("key"));
  assertEquals("2", record.get("intval"));
  assertNull(record.get("null"));
  assertEquals("defval", record.get("null", "defval"));
  byte[] bytes = marshal.toBytes(record);
  ServiceRecord r2 = marshal.fromBytes("", bytes);
  assertEquals("value", r2.get("key"));
  assertEquals("2", r2.get("intval"));
}

Source File: TestMarshalling.java From hadoop with Apache License 2.0

5 votes

@Test
public void testRoundTrip() throws Throwable {
  String persistence = PersistencePolicies.PERMANENT;
  ServiceRecord record = createRecord(persistence);
  record.set("customkey", "customvalue");
  record.set("customkey2", "customvalue2");
  RegistryTypeUtils.validateServiceRecord("", record);
  LOG.info(marshal.toJson(record));
  byte[] bytes = marshal.toBytes(record);
  ServiceRecord r2 = marshal.fromBytes("", bytes);
  assertMatches(record, r2);
  RegistryTypeUtils.validateServiceRecord("", r2);
}

Source File: TestRegistryOperations.java From hadoop with Apache License 2.0

5 votes

@Test
public void testPutNoParent() throws Throwable {
  ServiceRecord record = new ServiceRecord();
  record.set(YarnRegistryAttributes.YARN_ID, "testPutNoParent");
  String path = "/path/without/parent";
  try {
    operations.bind(path, record, 0);
    // didn't get a failure
    // trouble
    RegistryPathStatus stat = operations.stat(path);
    fail("Got a status " + stat);
  } catch (PathNotFoundException expected) {
    // expected
  }
}

Source File: TestRegistryRMOperations.java From big-c with Apache License 2.0

4 votes

@Test
public void testPurgeEntryCuratorCallback() throws Throwable {

  String path = "/users/example/hbase/hbase1/";
  ServiceRecord written = buildExampleServiceEntry(
      PersistencePolicies.APPLICATION_ATTEMPT);
  written.set(YarnRegistryAttributes.YARN_ID,
      "testAsyncPurgeEntry_attempt_001");

  operations.mknode(RegistryPathUtils.parentOf(path), true);
  operations.bind(path, written, 0);

  ZKPathDumper dump = registry.dumpPath(false);
  CuratorEventCatcher events = new CuratorEventCatcher();

  LOG.info("Initial state {}", dump);

  // container query
  String id = written.get(YarnRegistryAttributes.YARN_ID, "");
  int opcount = purge("/",
      id,
      PersistencePolicies.CONTAINER,
      RegistryAdminService.PurgePolicy.PurgeAll,
      events);
  assertPathExists(path);
  assertEquals(0, opcount);
  assertEquals("Event counter", 0, events.getCount());

  // now the application attempt
  opcount = purge("/",
      id,
      PersistencePolicies.APPLICATION_ATTEMPT,
      RegistryAdminService.PurgePolicy.PurgeAll,
      events);

  LOG.info("Final state {}", dump);

  assertPathNotFound(path);
  assertEquals("wrong no of delete operations in " + dump, 1, opcount);
  // and validate the callback event
  assertEquals("Event counter", 1, events.getCount());
}

Source File: TestRegistryRMOperations.java From big-c with Apache License 2.0

4 votes

@Test
public void testAsyncPurgeEntry() throws Throwable {

  String path = "/users/example/hbase/hbase1/";
  ServiceRecord written = buildExampleServiceEntry(
      PersistencePolicies.APPLICATION_ATTEMPT);
  written.set(YarnRegistryAttributes.YARN_ID,
      "testAsyncPurgeEntry_attempt_001");

  operations.mknode(RegistryPathUtils.parentOf(path), true);
  operations.bind(path, written, 0);

  ZKPathDumper dump = registry.dumpPath(false);

  LOG.info("Initial state {}", dump);

  DeleteCompletionCallback deletions = new DeleteCompletionCallback();
  int opcount = purge("/",
      written.get(YarnRegistryAttributes.YARN_ID, ""),
      PersistencePolicies.CONTAINER,
      RegistryAdminService.PurgePolicy.PurgeAll,
      deletions);
  assertPathExists(path);

  dump = registry.dumpPath(false);

  assertEquals("wrong no of delete operations in " + dump, 0,
      deletions.getEventCount());
  assertEquals("wrong no of delete operations in " + dump, 0, opcount);


  // now app attempt
  deletions = new DeleteCompletionCallback();
  opcount = purge("/",
      written.get(YarnRegistryAttributes.YARN_ID, ""),
      PersistencePolicies.APPLICATION_ATTEMPT,
      RegistryAdminService.PurgePolicy.PurgeAll,
      deletions);

  dump = registry.dumpPath(false);
  LOG.info("Final state {}", dump);

  assertPathNotFound(path);
  assertEquals("wrong no of delete operations in " + dump, 1,
      deletions.getEventCount());
  assertEquals("wrong no of delete operations in " + dump, 1, opcount);
  // and validate the callback event

}

Source File: TestRegistryRMOperations.java From hadoop with Apache License 2.0

4 votes

@Test
public void testAsyncPurgeEntry() throws Throwable {

  String path = "/users/example/hbase/hbase1/";
  ServiceRecord written = buildExampleServiceEntry(
      PersistencePolicies.APPLICATION_ATTEMPT);
  written.set(YarnRegistryAttributes.YARN_ID,
      "testAsyncPurgeEntry_attempt_001");

  operations.mknode(RegistryPathUtils.parentOf(path), true);
  operations.bind(path, written, 0);

  ZKPathDumper dump = registry.dumpPath(false);

  LOG.info("Initial state {}", dump);

  DeleteCompletionCallback deletions = new DeleteCompletionCallback();
  int opcount = purge("/",
      written.get(YarnRegistryAttributes.YARN_ID, ""),
      PersistencePolicies.CONTAINER,
      RegistryAdminService.PurgePolicy.PurgeAll,
      deletions);
  assertPathExists(path);

  dump = registry.dumpPath(false);

  assertEquals("wrong no of delete operations in " + dump, 0,
      deletions.getEventCount());
  assertEquals("wrong no of delete operations in " + dump, 0, opcount);


  // now app attempt
  deletions = new DeleteCompletionCallback();
  opcount = purge("/",
      written.get(YarnRegistryAttributes.YARN_ID, ""),
      PersistencePolicies.APPLICATION_ATTEMPT,
      RegistryAdminService.PurgePolicy.PurgeAll,
      deletions);

  dump = registry.dumpPath(false);
  LOG.info("Final state {}", dump);

  assertPathNotFound(path);
  assertEquals("wrong no of delete operations in " + dump, 1,
      deletions.getEventCount());
  assertEquals("wrong no of delete operations in " + dump, 1, opcount);
  // and validate the callback event

}

Source File: TestRegistryRMOperations.java From hadoop with Apache License 2.0

4 votes

@Test
public void testPurgeEntryCuratorCallback() throws Throwable {

  String path = "/users/example/hbase/hbase1/";
  ServiceRecord written = buildExampleServiceEntry(
      PersistencePolicies.APPLICATION_ATTEMPT);
  written.set(YarnRegistryAttributes.YARN_ID,
      "testAsyncPurgeEntry_attempt_001");

  operations.mknode(RegistryPathUtils.parentOf(path), true);
  operations.bind(path, written, 0);

  ZKPathDumper dump = registry.dumpPath(false);
  CuratorEventCatcher events = new CuratorEventCatcher();

  LOG.info("Initial state {}", dump);

  // container query
  String id = written.get(YarnRegistryAttributes.YARN_ID, "");
  int opcount = purge("/",
      id,
      PersistencePolicies.CONTAINER,
      RegistryAdminService.PurgePolicy.PurgeAll,
      events);
  assertPathExists(path);
  assertEquals(0, opcount);
  assertEquals("Event counter", 0, events.getCount());

  // now the application attempt
  opcount = purge("/",
      id,
      PersistencePolicies.APPLICATION_ATTEMPT,
      RegistryAdminService.PurgePolicy.PurgeAll,
      events);

  LOG.info("Final state {}", dump);

  assertPathNotFound(path);
  assertEquals("wrong no of delete operations in " + dump, 1, opcount);
  // and validate the callback event
  assertEquals("Event counter", 1, events.getCount());
}

Source File: JstormEngineServiceImpl.java From PoseidonX with Apache License 2.0

4 votes

public static void stopTask(Integer taskId, String taskName, Integer processId) throws Exception {
    if(taskId == null){
        throw new Exception("任务停止失败，参数异常，taskId为空！");
    }
    if(StringUtils.isBlank(taskName)){
        throw new Exception("任务提交失败，参数异常，任务名为空！");
    }
    if(processId == null){
        throw new Exception("任务停止失败，参数异常，processId为空！");
    }
    JstormProcessPO jstormProcessPO = jstormEngineService.jstormProcessDao.getById(processId);
    if(jstormProcessPO == null){
        throw new Exception("任务停止失败，任务的执行信息不存在或者已经被删除！");
    }
    JstormTaskConfigDTO jstormTaskConfigDTO = JSONObject.parseObject(jstormProcessPO.getTaskConfig(), JstormTaskConfigDTO.class);

    String topId = jstormProcessPO.getTopId();
    String appId = jstormProcessPO.getYarnAppId();
    String applicationPath = YarnZkRegistryBusiness.PathBuilder.applicationPath(jstormTaskConfigDTO.getTaskName(),jstormProcessPO.getYarnAppId());
    ServiceRecord serviceRecord  = YarnZkRegistryBusiness.resolve(applicationPath);
    if(serviceRecord != null){
        // 设置为true
        serviceRecord.set("killed","true");
        YarnZkRegistryBusiness.bind(applicationPath,serviceRecord);
    }

    ApplicationReport report = YarnClientProxy.getApplicationReportByAppId(appId);
    if(report != null){
        YarnApplicationState state = report.getYarnApplicationState();

        LOGGER.error("JstormProcessServiceImpl stopTask report YarnApplicationState = " + state.toString() + " appId=" + appId);

        if (!(YarnApplicationState.FINISHED == state || YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state)) {
            try{
                YarnClientProxy.killApplicationByAppId(appId);
                LOGGER.error("JstormProcessServiceImpl stopTask killApplicationByAppId is ok appId=" + appId);
            }catch (Exception e){
                // kill失败时进行还原处理
                if(serviceRecord != null){
                    serviceRecord.set("killed","false");
                    YarnZkRegistryBusiness.bind(applicationPath,serviceRecord);
                }
                LOGGER.error("JstormProcessServiceImpl stopTask killApplicationByAppId is error appId=" + appId,e);
                throw e;
            }
        }
    }else{
        LOGGER.error("JstormProcessServiceImpl stopTask report is null appId=" + appId);
    }

    TaskPO taskPO = new TaskPO();
    taskPO.setTaskStatus(TaskStatusEnum.STOP.getValue());
    taskPO.setTaskStopTime(new Date());
    taskPO.setId(jstormProcessPO.getTaskId());
    jstormEngineService.taskDao.update4Stop(taskPO);

    //清理资源
    JStormOnYarnBusiness.clearZkInfo(jstormTaskConfigDTO.getJstormZkHost(), jstormTaskConfigDTO.getJstormZkPort(), jstormTaskConfigDTO.getTaskName());
    JstormMetricCollectImpl.workerErrorHistoryCache.remove(topId);
    MetricReportContainer.removeReports(topId);
    JstormEngineCheckPointImpl.recoveryFail.remove(jstormProcessPO.getTaskId());
}

Java Code Examples for org.apache.hadoop.registry.client.types.ServiceRecord#set()