Java Code Examples for org.apache.hadoop.yarn.client.api.TimelineClient#createTimelineClient()

The following examples show how to use org.apache.hadoop.yarn.client.api.TimelineClient#createTimelineClient() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TimelineDelegationTokenIdentifier.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@SuppressWarnings("unchecked")
@Override
public long renew(Token<?> token, Configuration conf) throws IOException,
    InterruptedException {
  TimelineClient client = TimelineClient.createTimelineClient();
  try {
    client.init(conf);
    client.start();
    return client.renewDelegationToken(
        (Token<TimelineDelegationTokenIdentifier>) token);
  } catch (YarnException e) {
    throw new IOException(e);
  } finally {
    client.stop();
  }
}
Example 2
Source File: TimelineDelegationTokenIdentifier.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@SuppressWarnings("unchecked")
@Override
public void cancel(Token<?> token, Configuration conf) throws IOException,
    InterruptedException {
  TimelineClient client = TimelineClient.createTimelineClient();
  try {
    client.init(conf);
    client.start();
    client.cancelDelegationToken(
        (Token<TimelineDelegationTokenIdentifier>) token);
  } catch (YarnException e) {
    throw new IOException(e);
  } finally {
    client.stop();
  }
}
Example 3
Source File: SystemMetricsPublisher.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Override
protected void serviceInit(Configuration conf) throws Exception {
  publishSystemMetrics =
      conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
          YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED) &&
      conf.getBoolean(YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_ENABLED,
          YarnConfiguration.DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_ENABLED);

  if (publishSystemMetrics) {
    client = TimelineClient.createTimelineClient();
    addIfService(client);

    dispatcher = createDispatcher(conf);
    dispatcher.register(SystemMetricsEventType.class,
        new ForwardingEventHandler());
    addIfService(dispatcher);
    LOG.info("YARN system metrics publishing service is enabled");
  } else {
    LOG.info("YARN system metrics publishing service is not enabled");
  }
  super.serviceInit(conf);
}
Example 4
Source File: YarnModule.java    From sylph with Apache License 2.0 6 votes vote down vote up 
@Override
public YarnClient get()
{
    YarnClient client = YarnClient.createYarnClient();
    if (yarnConfiguration.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false)) {
        try {
            TimelineClient.createTimelineClient();
        }
        catch (NoClassDefFoundError e) {
            logger.warn("createTimelineClient() error with {}", TimelineClient.class.getResource(TimelineClient.class.getSimpleName() + ".class"), e);
            yarnConfiguration.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false);
        }
    }
    client.init(yarnConfiguration);
    client.start();
    return client;
}
Example 5
Source File: TimelineDelegationTokenIdentifier.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@SuppressWarnings("unchecked")
@Override
public long renew(Token<?> token, Configuration conf) throws IOException,
    InterruptedException {
  TimelineClient client = TimelineClient.createTimelineClient();
  try {
    client.init(conf);
    client.start();
    return client.renewDelegationToken(
        (Token<TimelineDelegationTokenIdentifier>) token);
  } catch (YarnException e) {
    throw new IOException(e);
  } finally {
    client.stop();
  }
}
Example 6
Source File: TimelineDelegationTokenIdentifier.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@SuppressWarnings("unchecked")
@Override
public void cancel(Token<?> token, Configuration conf) throws IOException,
    InterruptedException {
  TimelineClient client = TimelineClient.createTimelineClient();
  try {
    client.init(conf);
    client.start();
    client.cancelDelegationToken(
        (Token<TimelineDelegationTokenIdentifier>) token);
  } catch (YarnException e) {
    throw new IOException(e);
  } finally {
    client.stop();
  }
}
Example 7
Source File: SystemMetricsPublisher.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Override
protected void serviceInit(Configuration conf) throws Exception {
  publishSystemMetrics =
      conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
          YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED) &&
      conf.getBoolean(YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_ENABLED,
          YarnConfiguration.DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_ENABLED);

  if (publishSystemMetrics) {
    client = TimelineClient.createTimelineClient();
    addIfService(client);

    dispatcher = createDispatcher(conf);
    dispatcher.register(SystemMetricsEventType.class,
        new ForwardingEventHandler());
    addIfService(dispatcher);
    LOG.info("YARN system metrics publishing service is enabled");
  } else {
    LOG.info("YARN system metrics publishing service is not enabled");
  }
  super.serviceInit(conf);
}
Example 8
Source File: JstormOnYarn.java    From jstorm with Apache License 2.0 5 votes vote down vote up 
private void prepareTimelineDomain() {
    TimelineClient timelineClient = null;
    if (jstormClientContext.conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
            YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
        timelineClient = TimelineClient.createTimelineClient();
        timelineClient.init(jstormClientContext.conf);
        timelineClient.start();
    } else {
        LOG.warn("Cannot put the domain " + jstormClientContext.domainId +
                " because the timeline service is not enabled");
        return;
    }
    try {
        TimelineDomain domain = new TimelineDomain();
        domain.setId(jstormClientContext.domainId);
        domain.setReaders(
                jstormClientContext.viewACLs != null && jstormClientContext.viewACLs.length() > 0 ? jstormClientContext.viewACLs : JOYConstants.BLANK);
        domain.setWriters(
                jstormClientContext.modifyACLs != null && jstormClientContext.modifyACLs.length() > 0 ? jstormClientContext.modifyACLs : JOYConstants.BLANK);
        timelineClient.putDomain(domain);
        LOG.info("Put the timeline domain: " +
                TimelineUtils.dumpTimelineRecordtoJSON(domain));
    } catch (Exception e) {
        LOG.error("Error when putting the timeline domain", e);
    } finally {
        timelineClient.stop();
    }
}
Example 9
Source File: ATSHistoryACLPolicyManager.java    From tez with Apache License 2.0 5 votes vote down vote up 
private void initializeTimelineClient() {
  if (this.conf == null) {
    throw new TezUncheckedException("ATSACLManager not configured");
  }
  if (timelineClient != null) {
    this.timelineClient.stop();
    this.timelineClient = null;
  }
  if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
    YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
    this.timelineClient = TimelineClient.createTimelineClient();
    this.timelineClient.init(this.conf);
    this.timelineClient.start();
  } else {
    this.timelineClient = null;
    if (conf.get(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, "")
       .equals(atsHistoryLoggingServiceClassName)) {
      LOG.warn(atsHistoryLoggingServiceClassName
          + " is disabled due to Timeline Service being disabled, "
          + YarnConfiguration.TIMELINE_SERVICE_ENABLED + " set to false");
    }
  }
  try {
    this.user = UserGroupInformation.getCurrentUser().getShortUserName();
  } catch (IOException e) {
    throw new TezUncheckedException("Unable to get Current User UGI", e);
  }
}
Example 10
Source File: ATSV15HistoryACLPolicyManager.java    From tez with Apache License 2.0 5 votes vote down vote up 
private void initializeTimelineClient() {
  if (this.conf == null) {
    throw new TezUncheckedException("ATSACLManager not configured");
  }
  if (timelineClient != null) {
    this.timelineClient.stop();
    this.timelineClient = null;
  }
  if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
    YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
    this.timelineClient = TimelineClient.createTimelineClient();
    this.timelineClient.init(this.conf);
    this.timelineClient.start();
  } else {
    this.timelineClient = null;
    if (conf.get(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, "")
       .equals(atsHistoryLoggingServiceClassName)) {
      LOG.warn(atsHistoryLoggingServiceClassName
          + " is disabled due to Timeline Service being disabled, "
          + YarnConfiguration.TIMELINE_SERVICE_ENABLED + " set to false");
    }
  }
  try {
    this.user = UserGroupInformation.getCurrentUser().getShortUserName();
  } catch (IOException e) {
    throw new TezUncheckedException("Unable to get Current User UGI", e);
  }
}
Example 11
Source File: ATSHistoryLoggingService.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up 
@Override
public void serviceInit(Configuration conf) throws Exception {
  LOG.info("Initializing ATSService");
  timelineClient = TimelineClient.createTimelineClient();
  timelineClient.init(conf);
  maxTimeToWaitOnShutdown = conf.getLong(
      TezConfiguration.YARN_ATS_EVENT_FLUSH_TIMEOUT_MILLIS,
      TezConfiguration.YARN_ATS_EVENT_FLUSH_TIMEOUT_MILLIS_DEFAULT);
}
Example 12
Source File: Client.java    From metron with Apache License 2.0 5 votes vote down vote up 
private void prepareTimelineDomain() {
  TimelineClient timelineClient = null;
  if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
          YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
    timelineClient = TimelineClient.createTimelineClient();
    timelineClient.init(conf);
    timelineClient.start();
  } else {
    LOG.warn("Cannot put the domain " + domainId +
            " because the timeline service is not enabled");
    return;
  }
  try {
    TimelineDomain domain = new TimelineDomain();
    domain.setId(domainId);
    domain.setReaders(
            viewACLs != null && viewACLs.length() > 0 ? viewACLs : " ");
    domain.setWriters(
            modifyACLs != null && modifyACLs.length() > 0 ? modifyACLs : " ");
    timelineClient.putDomain(domain);
    LOG.info("Put the timeline domain: " +
            TimelineUtils.dumpTimelineRecordtoJSON(domain));
  } catch (Exception e) {
    LOG.error("Error when putting the timeline domain", e);
  } finally {
    timelineClient.stop();
  }
}
Example 13
Source File: Client.java    From big-c with Apache License 2.0 5 votes vote down vote up 
private void prepareTimelineDomain() {
  TimelineClient timelineClient = null;
  if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
      YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
    timelineClient = TimelineClient.createTimelineClient();
    timelineClient.init(conf);
    timelineClient.start();
  } else {
    LOG.warn("Cannot put the domain " + domainId +
        " because the timeline service is not enabled");
    return;
  }
  try {
    //TODO: we need to check and combine the existing timeline domain ACLs,
    //but let's do it once we have client java library to query domains.
    TimelineDomain domain = new TimelineDomain();
    domain.setId(domainId);
    domain.setReaders(
        viewACLs != null && viewACLs.length() > 0 ? viewACLs : " ");
    domain.setWriters(
        modifyACLs != null && modifyACLs.length() > 0 ? modifyACLs : " ");
    timelineClient.putDomain(domain);
    LOG.info("Put the timeline domain: " +
        TimelineUtils.dumpTimelineRecordtoJSON(domain));
  } catch (Exception e) {
    LOG.error("Error when putting the timeline domain", e);
  } finally {
    timelineClient.stop();
  }
}
Example 14
Source File: Client.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
private void prepareTimelineDomain() {
  TimelineClient timelineClient = null;
  if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
      YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
    timelineClient = TimelineClient.createTimelineClient();
    timelineClient.init(conf);
    timelineClient.start();
  } else {
    LOG.warn("Cannot put the domain " + domainId +
        " because the timeline service is not enabled");
    return;
  }
  try {
    //TODO: we need to check and combine the existing timeline domain ACLs,
    //but let's do it once we have client java library to query domains.
    TimelineDomain domain = new TimelineDomain();
    domain.setId(domainId);
    domain.setReaders(
        viewACLs != null && viewACLs.length() > 0 ? viewACLs : " ");
    domain.setWriters(
        modifyACLs != null && modifyACLs.length() > 0 ? modifyACLs : " ");
    timelineClient.putDomain(domain);
    LOG.info("Put the timeline domain: " +
        TimelineUtils.dumpTimelineRecordtoJSON(domain));
  } catch (Exception e) {
    LOG.error("Error when putting the timeline domain", e);
  } finally {
    timelineClient.stop();
  }
}
Example 15
Source File: YarnClientImpl.java    From big-c with Apache License 2.0 4 votes vote down vote up 
TimelineClient createTimelineClient() throws IOException, YarnException {
  return TimelineClient.createTimelineClient();
}
Example 16
Source File: TestTimelineAuthenticationFilter.java    From big-c with Apache License 2.0 4 votes vote down vote up 
private TimelineClient createTimelineClientForUGI() {
  TimelineClient client = TimelineClient.createTimelineClient();
  client.init(conf);
  client.start();
  return client;
}
Example 17
Source File: ATSV15HistoryLoggingService.java    From tez with Apache License 2.0 4 votes vote down vote up 
@Override
public void serviceInit(Configuration serviceConf) throws Exception {
  Configuration conf = new Configuration(serviceConf);

  String summaryEntityTypesStr = EntityTypes.TEZ_APPLICATION
      + "," + EntityTypes.TEZ_APPLICATION_ATTEMPT
      + "," + EntityTypes.TEZ_DAG_ID;

  // Ensure that summary entity types are defined properly for Tez.
  if (conf.getBoolean(TezConfiguration.TEZ_AM_ATS_V15_OVERRIDE_SUMMARY_TYPES,
      TezConfiguration.TEZ_AM_ATS_V15_OVERRIDE_SUMMARY_TYPES_DEFAULT)) {
    conf.set(YarnConfiguration.TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_SUMMARY_ENTITY_TYPES,
        summaryEntityTypesStr);
  }

  historyLoggingEnabled = conf.getBoolean(TezConfiguration.TEZ_AM_HISTORY_LOGGING_ENABLED,
      TezConfiguration.TEZ_AM_HISTORY_LOGGING_ENABLED_DEFAULT);
  if (!historyLoggingEnabled) {
    LOG.info("ATSService: History Logging disabled. "
        + TezConfiguration.TEZ_AM_HISTORY_LOGGING_ENABLED + " set to false");
    return;
  }

  if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
    YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
    timelineClient = TimelineClient.createTimelineClient();
    timelineClient.init(conf);
  } else {
    this.timelineClient = null;
    if (conf.get(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, "")
      .equals(atsHistoryLoggingServiceClassName)) {
      LOG.warn(atsHistoryLoggingServiceClassName
          + " is disabled due to Timeline Service being disabled, "
          + YarnConfiguration.TIMELINE_SERVICE_ENABLED + " set to false");
    }
  }
  maxTimeToWaitOnShutdown = conf.getLong(
      TezConfiguration.YARN_ATS_EVENT_FLUSH_TIMEOUT_MILLIS,
      TezConfiguration.YARN_ATS_EVENT_FLUSH_TIMEOUT_MILLIS_DEFAULT);
  maxPollingTimeMillis = conf.getInt(
      TezConfiguration.YARN_ATS_MAX_POLLING_TIME_PER_EVENT,
      TezConfiguration.YARN_ATS_MAX_POLLING_TIME_PER_EVENT_DEFAULT);
  if (maxTimeToWaitOnShutdown < 0) {
    waitForeverOnShutdown = true;
  }

  LOG.info("Initializing " + ATSV15HistoryLoggingService.class.getSimpleName() + " with "
      + ", maxPollingTime(ms)=" + maxPollingTimeMillis
      + ", waitTimeForShutdown(ms)=" + maxTimeToWaitOnShutdown
      + ", TimelineACLManagerClass=" + atsHistoryACLManagerClassName);

  try {
    historyACLPolicyManager = ReflectionUtils.createClazzInstance(
        atsHistoryACLManagerClassName);
    historyACLPolicyManager.setConf(conf);
  } catch (TezReflectionException e) {
    LOG.warn("Could not instantiate object for " + atsHistoryACLManagerClassName
        + ". ACLs cannot be enforced correctly for history data in Timeline", e);
    if (!conf.getBoolean(TezConfiguration.TEZ_AM_ALLOW_DISABLED_TIMELINE_DOMAINS,
        TezConfiguration.TEZ_AM_ALLOW_DISABLED_TIMELINE_DOMAINS_DEFAULT)) {
      throw e;
    }
    historyACLPolicyManager = null;
  }

  numDagsPerGroup = conf.getInt(TezConfiguration.TEZ_HISTORY_LOGGING_TIMELINE_NUM_DAGS_PER_GROUP,
      TezConfiguration.TEZ_HISTORY_LOGGING_TIMELINE_NUM_DAGS_PER_GROUP_DEFAULT);
}
Example 18
Source File: ATSHistoryLoggingService.java    From tez with Apache License 2.0 4 votes vote down vote up 
@Override
public void serviceInit(Configuration conf) throws Exception {
  historyLoggingEnabled = conf.getBoolean(TezConfiguration.TEZ_AM_HISTORY_LOGGING_ENABLED,
      TezConfiguration.TEZ_AM_HISTORY_LOGGING_ENABLED_DEFAULT);
  if (!historyLoggingEnabled) {
    LOG.info("ATSService: History Logging disabled. "
        + TezConfiguration.TEZ_AM_HISTORY_LOGGING_ENABLED + " set to false");
    return;
  }

  if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
    YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
    timelineClient = TimelineClient.createTimelineClient();
    timelineClient.init(conf);
  } else {
    this.timelineClient = null;
    if (conf.get(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, "")
      .equals(atsHistoryLoggingServiceClassName)) {
      LOG.warn(atsHistoryLoggingServiceClassName
          + " is disabled due to Timeline Service being disabled, "
          + YarnConfiguration.TIMELINE_SERVICE_ENABLED + " set to false");
    }
  }
  maxTimeToWaitOnShutdown = conf.getLong(
      TezConfiguration.YARN_ATS_EVENT_FLUSH_TIMEOUT_MILLIS,
      TezConfiguration.YARN_ATS_EVENT_FLUSH_TIMEOUT_MILLIS_DEFAULT);
  maxEventsPerBatch = conf.getInt(
      TezConfiguration.YARN_ATS_MAX_EVENTS_PER_BATCH,
      TezConfiguration.YARN_ATS_MAX_EVENTS_PER_BATCH_DEFAULT);
  maxPollingTimeMillis = conf.getInt(
      TezConfiguration.YARN_ATS_MAX_POLLING_TIME_PER_EVENT,
      TezConfiguration.YARN_ATS_MAX_POLLING_TIME_PER_EVENT_DEFAULT);
  if (maxTimeToWaitOnShutdown < 0) {
    waitForeverOnShutdown = true;
  }

  LOG.info("Initializing " + ATSHistoryLoggingService.class.getSimpleName() + " with "
    + "maxEventsPerBatch=" + maxEventsPerBatch
    + ", maxPollingTime(ms)=" + maxPollingTimeMillis
    + ", waitTimeForShutdown(ms)=" + maxTimeToWaitOnShutdown
    + ", TimelineACLManagerClass=" + atsHistoryACLManagerClassName);

  try {
    historyACLPolicyManager = ReflectionUtils.createClazzInstance(
        atsHistoryACLManagerClassName);
    historyACLPolicyManager.setConf(conf);
  } catch (TezReflectionException e) {
    LOG.warn("Could not instantiate object for " + atsHistoryACLManagerClassName
        + ". ACLs cannot be enforced correctly for history data in Timeline", e);
    if (!conf.getBoolean(TezConfiguration.TEZ_AM_ALLOW_DISABLED_TIMELINE_DOMAINS,
        TezConfiguration.TEZ_AM_ALLOW_DISABLED_TIMELINE_DOMAINS_DEFAULT)) {
      throw e;
    }
    historyACLPolicyManager = null;
  }

}
Example 19
Source File: YarnClientImpl.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
TimelineClient createTimelineClient() throws IOException, YarnException {
  return TimelineClient.createTimelineClient();
}
Example 20
Source File: TestTimelineAuthenticationFilter.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
private TimelineClient createTimelineClientForUGI() {
  TimelineClient client = TimelineClient.createTimelineClient();
  client.init(conf);
  client.start();
  return client;
}