org.apache.hadoop.registry.client.types.ServiceRecord Java Examples

@Test
public void testChildDeletion() throws Throwable {
  ServiceRecord app = createRecord("app1",
      PersistencePolicies.APPLICATION, "app",
      null);
  ServiceRecord container = createRecord("container1",
      PersistencePolicies.CONTAINER, "container",
      null);

  operations.bind("/app", app, BindFlags.OVERWRITE);
  operations.bind("/app/container", container, BindFlags.OVERWRITE);

  try {
    int p = purge("/",
        "app1",
        PersistencePolicies.APPLICATION,
        RegistryAdminService.PurgePolicy.FailOnChildren);
    fail("expected a failure, got a purge count of " + p);
  } catch (PathIsNotEmptyDirectoryException expected) {
    // expected
  }

}
 

/**
 * 删除应用
 * @param zkHosts
 * @param zkPort
 * @param appId
 */
public static void killApplication(List<String> zkHosts, Integer zkPort, String appId, String taskName) throws Exception{
    if(StringUtils.isNotBlank(appId) && StringUtils.isNotBlank(taskName)){

        String applicationPath = YarnZkRegistryBusiness.PathBuilder.applicationPath(taskName,appId);
        ServiceRecord serviceRecord = YarnZkRegistryBusiness.resolve(applicationPath);
        if(serviceRecord !=null){
            serviceRecord.set("killed","true");
            YarnZkRegistryBusiness.bind(applicationPath,serviceRecord);
        }
        ApplicationReport applicationReport = YarnClientProxy.getApplicationReportByAppId(appId);
        if(applicationReport !=null){
            try{
                YarnClientProxy.killApplicationByAppId(appId);
            }catch (Exception e){
                if(serviceRecord != null){
                    serviceRecord.set("killed","false");
                    YarnZkRegistryBusiness.bind(applicationPath,serviceRecord);
                }
                throw e;
            }
        }
    }
    JStormOnYarnBusiness.clearZkInfo(zkHosts,zkPort,taskName);
}
 

@Test
public void testLsParent() throws Throwable {
  ServiceRecord written = putExampleServiceEntry(ENTRY_PATH, 0);
  RegistryPathStatus stat = operations.stat(ENTRY_PATH);

  List<String> children = operations.list(PARENT_PATH);
  assertEquals(1, children.size());
  assertEquals(NAME, children.get(0));
  Map<String, RegistryPathStatus> childStats =
      RegistryUtils.statChildren(operations, PARENT_PATH);
  assertEquals(1, childStats.size());
  assertEquals(stat, childStats.get(NAME));

  Map<String, ServiceRecord> records =
      RegistryUtils.extractServiceRecords(operations,
          PARENT_PATH,
          childStats.values());
  assertEquals(1, records.size());
  ServiceRecord record = records.get(ENTRY_PATH);
  RegistryTypeUtils.validateServiceRecord(ENTRY_PATH, record);
  assertMatches(written, record);
}
 

/**
 * Assert the records match.
 * @param source record that was written
 * @param resolved the one that resolved.
 */
public static void assertMatches(ServiceRecord source, ServiceRecord resolved) {
  assertNotNull("Null source record ", source);
  assertNotNull("Null resolved record ", resolved);
  assertEquals(source.description, resolved.description);

  Map<String, String> srcAttrs = source.attributes();
  Map<String, String> resolvedAttrs = resolved.attributes();
  String sourceAsString = source.toString();
  String resolvedAsString = resolved.toString();
  assertEquals("Wrong count of attrs in \n" + sourceAsString
               + "\nfrom\n" + resolvedAsString,
      srcAttrs.size(),
      resolvedAttrs.size());
  for (Map.Entry<String, String> entry : srcAttrs.entrySet()) {
    String attr = entry.getKey();
    assertEquals("attribute "+ attr, entry.getValue(), resolved.get(attr));
  }
  assertEquals("wrong external endpoint count",
      source.external.size(), resolved.external.size());
  assertEquals("wrong external endpoint count",
      source.internal.size(), resolved.internal.size());
}
 

/**
 * Assert the records match.
 * @param source record that was written
 * @param resolved the one that resolved.
 */
public static void assertMatches(ServiceRecord source, ServiceRecord resolved) {
  assertNotNull("Null source record ", source);
  assertNotNull("Null resolved record ", resolved);
  assertEquals(source.description, resolved.description);

  Map<String, String> srcAttrs = source.attributes();
  Map<String, String> resolvedAttrs = resolved.attributes();
  String sourceAsString = source.toString();
  String resolvedAsString = resolved.toString();
  assertEquals("Wrong count of attrs in \n" + sourceAsString
               + "\nfrom\n" + resolvedAsString,
      srcAttrs.size(),
      resolvedAttrs.size());
  for (Map.Entry<String, String> entry : srcAttrs.entrySet()) {
    String attr = entry.getKey();
    assertEquals("attribute "+ attr, entry.getValue(), resolved.get(attr));
  }
  assertEquals("wrong external endpoint count",
      source.external.size(), resolved.external.size());
  assertEquals("wrong external endpoint count",
      source.internal.size(), resolved.internal.size());
}
 

@Test
public void testOverwrite() throws Throwable {
  ServiceRecord written = putExampleServiceEntry(ENTRY_PATH, 0);
  ServiceRecord resolved1 = operations.resolve(ENTRY_PATH);
  resolved1.description = "resolved1";
  try {
    operations.bind(ENTRY_PATH, resolved1, 0);
    fail("overwrite succeeded when it should have failed");
  } catch (FileAlreadyExistsException expected) {
    // expected
  }

  // verify there's no changed
  ServiceRecord resolved2 = operations.resolve(ENTRY_PATH);
  assertMatches(written, resolved2);
  operations.bind(ENTRY_PATH, resolved1, BindFlags.OVERWRITE);
  ServiceRecord resolved3 = operations.resolve(ENTRY_PATH);
  assertMatches(resolved1, resolved3);
}
 

/**
 * Find an endpoint in a record or fail,
 * @param record record
 * @param api API
 * @param external external?
 * @param addressElements expected # of address elements?
 * @param addressTupleSize expected size of a type
 * @return the endpoint.
 */
public static Endpoint findEndpoint(ServiceRecord record,
    String api, boolean external, int addressElements, int addressTupleSize) {
  Endpoint epr = external ? record.getExternalEndpoint(api)
                          : record.getInternalEndpoint(api);
  if (epr != null) {
    assertEquals("wrong # of addresses",
        addressElements, epr.addresses.size());
    assertEquals("wrong # of elements in an address tuple",
        addressTupleSize, epr.addresses.get(0).size());
    return epr;
  }
  List<Endpoint> endpoints = external ? record.external : record.internal;
  StringBuilder builder = new StringBuilder();
  for (Endpoint endpoint : endpoints) {
    builder.append("\"").append(endpoint).append("\" ");
  }
  fail("Did not find " + api + " in endpoints " + builder);
  // never reached; here to keep the compiler happy
  return null;
}
 

/**
 * Add some endpoints
 * @param entry entry
 */
public static void addSampleEndpoints(ServiceRecord entry, String hostname)
    throws URISyntaxException {
  assertNotNull(hostname);
  entry.addExternalEndpoint(webEndpoint(HTTP_API,
      new URI("http", hostname + ":80", "/")));
  entry.addExternalEndpoint(
      restEndpoint(API_WEBHDFS,
          new URI("http", hostname + ":8020", "/")));

  Endpoint endpoint = ipcEndpoint(API_HDFS, null);
  endpoint.addresses.add(RegistryTypeUtils.hostnamePortPair(hostname, 8030));
  entry.addInternalEndpoint(endpoint);
  InetSocketAddress localhost = new InetSocketAddress("localhost", 8050);
  entry.addInternalEndpoint(
      inetAddrEndpoint(NNIPC, ProtocolTypes.PROTOCOL_THRIFT, "localhost",
          8050));
  entry.addInternalEndpoint(
      RegistryTypeUtils.ipcEndpoint(
          IPC2, localhost));
}
 

public boolean needUpgrade() {
    String containerPath = RegistryUtils.componentPath(
            JOYConstants.APP_TYPE, this.executorMeta.getInstanceName(),
            this.executorMeta.getApplicationId(), this.executorMeta.getRunningContainer());

    try {
        if (registryOperations.exists(containerPath)) {
            ServiceRecord sr = registryOperations.resolve(containerPath);
            if (sr.get(JOYConstants.NEED_UPGRADE) != null && sr.get(JOYConstants.NEED_UPGRADE).equals(JOYConstants.TRUE)) {
                sr.set(JOYConstants.NEED_UPGRADE, JOYConstants.FALSE);
                registryOperations.bind(containerPath, sr, BindFlags.OVERWRITE);
                LOG.info(JOYConstants.NEED_UPGRADE);
                return true;
            }
        }
    } catch (IOException e) {
        e.printStackTrace();
    }
    return false;
}
 

@Test
public void testOverwrite() throws Throwable {
  ServiceRecord written = putExampleServiceEntry(ENTRY_PATH, 0);
  ServiceRecord resolved1 = operations.resolve(ENTRY_PATH);
  resolved1.description = "resolved1";
  try {
    operations.bind(ENTRY_PATH, resolved1, 0);
    fail("overwrite succeeded when it should have failed");
  } catch (FileAlreadyExistsException expected) {
    // expected
  }

  // verify there's no changed
  ServiceRecord resolved2 = operations.resolve(ENTRY_PATH);
  assertMatches(written, resolved2);
  operations.bind(ENTRY_PATH, resolved1, BindFlags.OVERWRITE);
  ServiceRecord resolved3 = operations.resolve(ENTRY_PATH);
  assertMatches(resolved1, resolved3);
}
 

@Test
public void testLsParent() throws Throwable {
  ServiceRecord written = putExampleServiceEntry(ENTRY_PATH, 0);
  RegistryPathStatus stat = operations.stat(ENTRY_PATH);

  List<String> children = operations.list(PARENT_PATH);
  assertEquals(1, children.size());
  assertEquals(NAME, children.get(0));
  Map<String, RegistryPathStatus> childStats =
      RegistryUtils.statChildren(operations, PARENT_PATH);
  assertEquals(1, childStats.size());
  assertEquals(stat, childStats.get(NAME));

  Map<String, ServiceRecord> records =
      RegistryUtils.extractServiceRecords(operations,
          PARENT_PATH,
          childStats.values());
  assertEquals(1, records.size());
  ServiceRecord record = records.get(ENTRY_PATH);
  RegistryTypeUtils.validateServiceRecord(ENTRY_PATH, record);
  assertMatches(written, record);
}
 

/**
 * Find an endpoint in a record or fail,
 * @param record record
 * @param api API
 * @param external external?
 * @param addressElements expected # of address elements?
 * @param addressTupleSize expected size of a type
 * @return the endpoint.
 */
public static Endpoint findEndpoint(ServiceRecord record,
    String api, boolean external, int addressElements, int addressTupleSize) {
  Endpoint epr = external ? record.getExternalEndpoint(api)
                          : record.getInternalEndpoint(api);
  if (epr != null) {
    assertEquals("wrong # of addresses",
        addressElements, epr.addresses.size());
    assertEquals("wrong # of elements in an address tuple",
        addressTupleSize, epr.addresses.get(0).size());
    return epr;
  }
  List<Endpoint> endpoints = external ? record.external : record.internal;
  StringBuilder builder = new StringBuilder();
  for (Endpoint endpoint : endpoints) {
    builder.append("\"").append(endpoint).append("\" ");
  }
  fail("Did not find " + api + " in endpoints " + builder);
  // never reached; here to keep the compiler happy
  return null;
}
 

@Test
public void testPutGetContainerPersistenceServiceEntry() throws Throwable {

  String path = ENTRY_PATH;
  ServiceRecord written = buildExampleServiceEntry(
      PersistencePolicies.CONTAINER);

  operations.mknode(RegistryPathUtils.parentOf(path), true);
  operations.bind(path, written, BindFlags.CREATE);
  ServiceRecord resolved = operations.resolve(path);
  validateEntry(resolved);
  assertMatches(written, resolved);
}
 

/**
 * see if anyone is updating host's port list, if not start , update this host itself
 * timeout is 45 seconds
 *
 * @param hostPath
 * @throws InterruptedException
 * @throws IOException
 */
private void tryHostLock(String hostPath) throws Exception {

    //if path has created 60 seconds ago, then delete
    if (registryOperations.exists(hostPath)) {
        try {
            ServiceRecord host = registryOperations.resolve(hostPath);
            Long cTime = Long.parseLong(host.get(JOYConstants.CTIME, JOYConstants.DEFAULT_CTIME));
            Date now = new Date();
            if (now.getTime() - cTime > JOYConstants.HOST_LOCK_TIMEOUT || cTime > now.getTime())
                registryOperations.delete(hostPath, true);
        } catch (Exception ex) {
            LOG.error(ex);
        }
    }

    int failedCount = JOYConstants.RETRY_TIMES;
    while (!registryOperations.mknode(hostPath, true)) {
        Thread.sleep(JOYConstants.SLEEP_INTERVAL);
        failedCount--;
        if (failedCount <= 0)
            break;
    }

    if (failedCount > 0) {
        ServiceRecord sr = new ServiceRecord();
        Date date = new Date();
        date.getTime();
        sr.set(JOYConstants.CTIME, String.valueOf(date.getTime()));
        registryOperations.bind(hostPath, sr, BindFlags.OVERWRITE);
        return;
    }
    throw new Exception("can't get host lock");
}
 

@Test(expected = PathNotFoundException.class)
public void testPutNoParent2() throws Throwable {
  ServiceRecord record = new ServiceRecord();
  record.set(YarnRegistryAttributes.YARN_ID, "testPutNoParent");
  String path = "/path/without/parent";
  operations.bind(path, record, 0);
}
 

@Test
public void testResolvePathThatHasNoEntry() throws Throwable {
  String empty = "/empty2";
  operations.mknode(empty, false);
  try {
    ServiceRecord record = operations.resolve(empty);
    fail("expected an exception, got " + record);
  } catch (NoRecordException expected) {
    // expected
  }
}
 

@Test
public void testRoundTrip() throws Throwable {
  String persistence = PersistencePolicies.PERMANENT;
  ServiceRecord record = createRecord(persistence);
  record.set("customkey", "customvalue");
  record.set("customkey2", "customvalue2");
  RegistryTypeUtils.validateServiceRecord("", record);
  LOG.info(marshal.toJson(record));
  byte[] bytes = marshal.toBytes(record);
  ServiceRecord r2 = marshal.fromBytes("", bytes);
  assertMatches(record, r2);
  RegistryTypeUtils.validateServiceRecord("", r2);
}
 

@Test(expected = NoRecordException.class)
public void testUnmarshallWrongLongType() throws Throwable {
  ServiceRecord record = new ServiceRecord();
  record.type = "ThisRecordHasALongButNonMatchingType";
  byte[] bytes = marshal.toBytes(record);
  ServiceRecord serviceRecord = marshal.fromBytes("marshalling",
      bytes, ServiceRecord.RECORD_TYPE);
}
 

public static ServiceRecord createRecord(String id, String persistence,
    String description) {
  ServiceRecord serviceRecord = new ServiceRecord();
  serviceRecord.set(YarnRegistryAttributes.YARN_ID, id);
  serviceRecord.description = description;
  serviceRecord.set(YarnRegistryAttributes.YARN_PERSISTENCE, persistence);
  return serviceRecord;
}
 

@Test
public void testPutNoParent() throws Throwable {
  ServiceRecord record = new ServiceRecord();
  record.set(YarnRegistryAttributes.YARN_ID, "testPutNoParent");
  String path = "/path/without/parent";
  try {
    operations.bind(path, record, 0);
    // didn't get a failure
    // trouble
    RegistryPathStatus stat = operations.stat(path);
    fail("Got a status " + stat);
  } catch (PathNotFoundException expected) {
    // expected
  }
}
 

@Override
public boolean shouldSelect(String path,
    RegistryPathStatus registryPathStatus,
    ServiceRecord serviceRecord) {
  String policy =
      serviceRecord.get(YarnRegistryAttributes.YARN_PERSISTENCE, "");
  return id.equals(serviceRecord.get(YarnRegistryAttributes.YARN_ID, ""))
         && (targetPolicy.equals(policy));
}
 

/**
 * List service records directly under a path
 * @param registryOperations registry operations instance
 * @param path path to list
 * @return a mapping of the service records that were resolved, indexed
 * by their full path
 * @throws IOException
 */
public static Map<String, ServiceRecord> listServiceRecords(
    RegistryOperations registryOperations,
    String path) throws IOException {
  Map<String, RegistryPathStatus> children =
      statChildren(registryOperations, path);
  return extractServiceRecords(registryOperations,
      path,
      children.values());
}
 

public static ServiceRecord createRecord(String id, String persistence,
    String description) {
  ServiceRecord serviceRecord = new ServiceRecord();
  serviceRecord.set(YarnRegistryAttributes.YARN_ID, id);
  serviceRecord.description = description;
  serviceRecord.set(YarnRegistryAttributes.YARN_PERSISTENCE, persistence);
  return serviceRecord;
}
 

/**
 * General code to validate bits of a component/service entry built iwth
 * {@link #addSampleEndpoints(ServiceRecord, String)}
 * @param record instance to check
 */
public static void validateEntry(ServiceRecord record) {
  assertNotNull("null service record", record);
  List<Endpoint> endpoints = record.external;
  assertEquals(2, endpoints.size());

  Endpoint webhdfs = findEndpoint(record, API_WEBHDFS, true, 1, 1);
  assertEquals(API_WEBHDFS, webhdfs.api);
  assertEquals(AddressTypes.ADDRESS_URI, webhdfs.addressType);
  assertEquals(ProtocolTypes.PROTOCOL_REST, webhdfs.protocolType);
  List<Map<String, String>> addressList = webhdfs.addresses;
  Map<String, String> url = addressList.get(0);
  String addr = url.get("uri");
  assertTrue(addr.contains("http"));
  assertTrue(addr.contains(":8020"));

  Endpoint nnipc = findEndpoint(record, NNIPC, false, 1,2);
  assertEquals("wrong protocol in " + nnipc, ProtocolTypes.PROTOCOL_THRIFT,
      nnipc.protocolType);

  Endpoint ipc2 = findEndpoint(record, IPC2, false, 1,2);
  assertNotNull(ipc2);

  Endpoint web = findEndpoint(record, HTTP_API, true, 1, 1);
  assertEquals(1, web.addresses.size());
  assertEquals(1, web.addresses.get(0).size());
}
 

/**
 * Create a service entry with the sample endpoints
 * @param persistence persistence policy
 * @return the record
 * @throws IOException on a failure
 */
public static ServiceRecord buildExampleServiceEntry(String persistence) throws
    IOException,
    URISyntaxException {
  ServiceRecord record = new ServiceRecord();
  record.set(YarnRegistryAttributes.YARN_ID, "example-0001");
  record.set(YarnRegistryAttributes.YARN_PERSISTENCE, persistence);
  addSampleEndpoints(record, "namenode");
  return record;
}
 

/**
 * Add a service under a path for the current user
 * @param record service record
 * @param deleteTreeFirst perform recursive delete of the path first
 * @return the path the service was created at
 * @throws IOException
 */
public String registerSelf(
    ServiceRecord record,
    boolean deleteTreeFirst) throws IOException {
  selfRegistrationPath =
      putService(user, jstormServiceClass, instanceName, record, deleteTreeFirst);
  setSelfRegistration(record);
  return selfRegistrationPath;
}
 

/**
 * General code to validate bits of a component/service entry built iwth
 * {@link #addSampleEndpoints(ServiceRecord, String)}
 * @param record instance to check
 */
public static void validateEntry(ServiceRecord record) {
  assertNotNull("null service record", record);
  List<Endpoint> endpoints = record.external;
  assertEquals(2, endpoints.size());

  Endpoint webhdfs = findEndpoint(record, API_WEBHDFS, true, 1, 1);
  assertEquals(API_WEBHDFS, webhdfs.api);
  assertEquals(AddressTypes.ADDRESS_URI, webhdfs.addressType);
  assertEquals(ProtocolTypes.PROTOCOL_REST, webhdfs.protocolType);
  List<Map<String, String>> addressList = webhdfs.addresses;
  Map<String, String> url = addressList.get(0);
  String addr = url.get("uri");
  assertTrue(addr.contains("http"));
  assertTrue(addr.contains(":8020"));

  Endpoint nnipc = findEndpoint(record, NNIPC, false, 1,2);
  assertEquals("wrong protocol in " + nnipc, ProtocolTypes.PROTOCOL_THRIFT,
      nnipc.protocolType);

  Endpoint ipc2 = findEndpoint(record, IPC2, false, 1,2);
  assertNotNull(ipc2);

  Endpoint web = findEndpoint(record, HTTP_API, true, 1, 1);
  assertEquals(1, web.addresses.size());
  assertEquals(1, web.addresses.get(0).size());
}
 

/**
 * Create a service entry with the sample endpoints, and put it
 * at the destination
 * @param path path
 * @param createFlags flags
 * @return the record
 * @throws IOException on a failure
 */
protected ServiceRecord putExampleServiceEntry(String path,
    int createFlags,
    String persistence)
    throws IOException, URISyntaxException {
  ServiceRecord record = buildExampleServiceEntry(persistence);

  registry.mknode(RegistryPathUtils.parentOf(path), true);
  operations.bind(path, record, createFlags);
  return record;
}
 

@Test
public void testFieldPropagationInCopy() throws Throwable {
  ServiceRecord record =
      createRecord(PersistencePolicies.APPLICATION_ATTEMPT);
  record.set("key", "value");
  record.set("intval", "2");
  ServiceRecord that = new ServiceRecord(record);
  assertMatches(record, that);
}
 

@Test
public void testUnknownFieldsRoundTrip() throws Throwable {
  ServiceRecord record =
      createRecord(PersistencePolicies.APPLICATION_ATTEMPT);
  record.set("key", "value");
  record.set("intval", "2");
  assertEquals("value", record.get("key"));
  assertEquals("2", record.get("intval"));
  assertNull(record.get("null"));
  assertEquals("defval", record.get("null", "defval"));
  byte[] bytes = marshal.toBytes(record);
  ServiceRecord r2 = marshal.fromBytes("", bytes);
  assertEquals("value", r2.get("key"));
  assertEquals("2", r2.get("intval"));
}