org.apache.hadoop.yarn.util.Clock Java Examples

public MockDAGAppMaster(ApplicationAttemptId applicationAttemptId, ContainerId containerId,
    String nmHost, int nmPort, int nmHttpPort, Clock clock, long appSubmitTime,
    boolean isSession, String workingDirectory, String[] localDirs, String[] logDirs,
    AtomicBoolean launcherGoFlag, boolean initFailFlag, boolean startFailFlag,
    Credentials credentials, String jobUserName, int handlerConcurrency, int numConcurrentContainers) {
  super(applicationAttemptId, containerId, nmHost, nmPort, nmHttpPort, clock, appSubmitTime,
      isSession, workingDirectory, localDirs, logDirs,  new TezApiVersionInfo().getVersion(),
      credentials, jobUserName, null);
  shutdownHandler = new MockDAGAppMasterShutdownHandler();
  this.launcherGoFlag = launcherGoFlag;
  this.initFailFlag = initFailFlag;
  this.startFailFlag = startFailFlag;
  Preconditions.checkArgument(handlerConcurrency > 0);
  this.handlerConcurrency = handlerConcurrency;
  this.numConcurrentContainers = numConcurrentContainers;
}
 

@Before
public void setUp() {
  clock = mock(Clock.class);
  plan = mock(Plan.class);
  rSystem = mock(ReservationSystem.class);
  plans.put(PLAN_NAME, plan);
  rrValidator = new ReservationInputValidator(clock);
  when(clock.getTime()).thenReturn(1L);
  ResourceCalculator rCalc = new DefaultResourceCalculator();
  Resource resource = Resource.newInstance(10240, 10);
  when(plan.getResourceCalculator()).thenReturn(rCalc);
  when(plan.getTotalCapacity()).thenReturn(resource);
  when(rSystem.getQueueForReservation(any(ReservationId.class))).thenReturn(
      PLAN_NAME);
  when(rSystem.getPlan(PLAN_NAME)).thenReturn(plan);
}
 

public DefaultSpeculator
    (Configuration conf, AppContext context,
     TaskRuntimeEstimator estimator, Clock clock) {
  super(DefaultSpeculator.class.getName());

  this.conf = conf;
  this.context = context;
  this.estimator = estimator;
  this.clock = clock;
  this.eventHandler = context.getEventHandler();
  this.soonestRetryAfterNoSpeculate =
      conf.getLong(MRJobConfig.SPECULATIVE_RETRY_AFTER_NO_SPECULATE,
              MRJobConfig.DEFAULT_SPECULATIVE_RETRY_AFTER_NO_SPECULATE);
  this.soonestRetryAfterSpeculate =
      conf.getLong(MRJobConfig.SPECULATIVE_RETRY_AFTER_SPECULATE,
              MRJobConfig.DEFAULT_SPECULATIVE_RETRY_AFTER_SPECULATE);
  this.proportionRunningTasksSpeculatable =
      conf.getDouble(MRJobConfig.SPECULATIVECAP_RUNNING_TASKS,
              MRJobConfig.DEFAULT_SPECULATIVECAP_RUNNING_TASKS);
  this.proportionTotalTasksSpeculatable =
      conf.getDouble(MRJobConfig.SPECULATIVECAP_TOTAL_TASKS,
              MRJobConfig.DEFAULT_SPECULATIVECAP_TOTAL_TASKS);
  this.minimumAllowedSpeculativeTasks =
      conf.getInt(MRJobConfig.SPECULATIVE_MINIMUM_ALLOWED_TASKS,
              MRJobConfig.DEFAULT_SPECULATIVE_MINIMUM_ALLOWED_TASKS);
}
 

@VisibleForTesting
void createHistoryDirs(Clock clock, long intervalCheckMillis,
    long timeOutMillis) throws IOException {
  long start = clock.getTime();
  boolean done = false;
  int counter = 0;
  while (!done &&
      ((timeOutMillis == -1) || (clock.getTime() - start < timeOutMillis))) {
    done = tryCreatingHistoryDirs(counter++ % 3 == 0); // log every 3 attempts, 30sec
    try {
      Thread.sleep(intervalCheckMillis);
    } catch (InterruptedException ex) {
      throw new YarnRuntimeException(ex);
    }
  }
  if (!done) {
    throw new YarnRuntimeException("Timed out '" + timeOutMillis+
            "ms' waiting for FileSystem to become available");
  }
}
 

public MRAppMaster(ApplicationAttemptId applicationAttemptId,
    ContainerId containerId, String nmHost, int nmPort, int nmHttpPort,
    Clock clock, long appSubmitTime) {
  super(MRAppMaster.class.getName());
  this.clock = clock;
  this.startTime = clock.getTime();
  this.appSubmitTime = appSubmitTime;
  this.appAttemptID = applicationAttemptId;
  this.containerID = containerId;
  this.nmHost = nmHost;
  this.nmPort = nmPort;
  this.nmHttpPort = nmHttpPort;
  this.metrics = MRAppMetrics.create();
  logSyncer = TaskLog.createLogSyncer();
  LOG.info("Created MRAppMaster for application " + applicationAttemptId);
}
 

InMemoryPlan(QueueMetrics queueMetrics, SharingPolicy policy,
    ReservationAgent agent, Resource totalCapacity, long step,
    ResourceCalculator resCalc, Resource minAlloc, Resource maxAlloc,
    String queueName, Planner replanner, boolean getMoveOnExpiry, Clock clock) {
  this.queueMetrics = queueMetrics;
  this.policy = policy;
  this.agent = agent;
  this.step = step;
  this.totalCapacity = totalCapacity;
  this.resCalc = resCalc;
  this.minAlloc = minAlloc;
  this.maxAlloc = maxAlloc;
  this.rleSparseVector = new RLESparseResourceAllocation(resCalc, minAlloc);
  this.queueName = queueName;
  this.replanner = replanner;
  this.getMoveOnExpiry = getMoveOnExpiry;
  this.clock = clock;
}
 

public DAGAppMaster(ApplicationAttemptId applicationAttemptId,
    ContainerId containerId, String nmHost, int nmPort, int nmHttpPort,
    Clock clock, long appSubmitTime, boolean isSession) {
  super(DAGAppMaster.class.getName());
  this.clock = clock;
  this.startTime = clock.getTime();
  this.appSubmitTime = appSubmitTime;
  this.appAttemptID = applicationAttemptId;
  this.containerID = containerId;
  this.nmHost = nmHost;
  this.nmPort = nmPort;
  this.nmHttpPort = nmHttpPort;
  this.state = DAGAppMasterState.NEW;
  this.isSession = isSession;
  // TODO Metrics
  //this.metrics = DAGAppMetrics.create();
  LOG.info("Created DAGAppMaster for application " + applicationAttemptId);
}
 

public DefaultSpeculator
    (Configuration conf, AppContext context,
     TaskRuntimeEstimator estimator, Clock clock) {
  super(DefaultSpeculator.class.getName());

  this.conf = conf;
  this.context = context;
  this.estimator = estimator;
  this.clock = clock;
  this.eventHandler = context.getEventHandler();
  this.soonestRetryAfterNoSpeculate =
      conf.getLong(MRJobConfig.SPECULATIVE_RETRY_AFTER_NO_SPECULATE,
              MRJobConfig.DEFAULT_SPECULATIVE_RETRY_AFTER_NO_SPECULATE);
  this.soonestRetryAfterSpeculate =
      conf.getLong(MRJobConfig.SPECULATIVE_RETRY_AFTER_SPECULATE,
              MRJobConfig.DEFAULT_SPECULATIVE_RETRY_AFTER_SPECULATE);
  this.proportionRunningTasksSpeculatable =
      conf.getDouble(MRJobConfig.SPECULATIVECAP_RUNNING_TASKS,
              MRJobConfig.DEFAULT_SPECULATIVECAP_RUNNING_TASKS);
  this.proportionTotalTasksSpeculatable =
      conf.getDouble(MRJobConfig.SPECULATIVECAP_TOTAL_TASKS,
              MRJobConfig.DEFAULT_SPECULATIVECAP_TOTAL_TASKS);
  this.minimumAllowedSpeculativeTasks =
      conf.getInt(MRJobConfig.SPECULATIVE_MINIMUM_ALLOWED_TASKS,
              MRJobConfig.DEFAULT_SPECULATIVE_MINIMUM_ALLOWED_TASKS);
}
 

private void setupPlanFollower() throws Exception {
  ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
  mClock = mock(Clock.class);
  mAgent = mock(ReservationAgent.class);

  String reservationQ = testUtil.getFullReservationQueueName();
  CapacitySchedulerConfiguration csConf = cs.getConfiguration();
  csConf.setReservationWindow(reservationQ, 20L);
  csConf.setMaximumCapacity(reservationQ, 40);
  csConf.setAverageCapacity(reservationQ, 20);
  policy.init(reservationQ, csConf);
}
 

private void setupPlanFollower() throws Exception {
  ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
  mClock = mock(Clock.class);
  mAgent = mock(ReservationAgent.class);

  String reservationQ = testUtil.getFullReservationQueueName();
  AllocationConfiguration allocConf = fs.getAllocationConfiguration();
  allocConf.setReservationWindow(20L);
  allocConf.setAverageCapacity(20);
  policy.init(reservationQ, allocConf);
}
 

private void setupPlanFollower() throws Exception {
  ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
  mClock = mock(Clock.class);
  mAgent = mock(ReservationAgent.class);

  String reservationQ = testUtil.getFullReservationQueueName();
  AllocationConfiguration allocConf = fs.getAllocationConfiguration();
  allocConf.setReservationWindow(20L);
  allocConf.setAverageCapacity(20);
  policy.init(reservationQ, allocConf);
}
 

public VertexImplWithControlledInitializerManager(TezVertexID vertexId,
                                                  VertexPlan vertexPlan, String vertexName,
                                                  Configuration conf,
                                                  EventHandler eventHandler,
                                                  TaskAttemptListener taskAttemptListener,
                                                  Clock clock, TaskHeartbeatHandler thh,
                                                  AppContext appContext,
                                                  VertexLocationHint vertexLocationHint,
                                                  DrainDispatcher dispatcher) {
  super(vertexId, vertexPlan, vertexName, conf, eventHandler,
      taskAttemptListener, clock, thh, true,
      appContext, vertexLocationHint, null, javaProfilerOptions);
  this.dispatcher = dispatcher;
}
 

@Before
@SuppressWarnings("unchecked")
public void setup() {
  conf = new Configuration(false);
  conf.setLong(WAIT_TIME_BEFORE_KILL, 10000);
  conf.setLong(MONITORING_INTERVAL, 3000);
  // report "ideal" preempt
  conf.setFloat(TOTAL_PREEMPTION_PER_ROUND, (float) 1.0);
  conf.setFloat(NATURAL_TERMINATION_FACTOR, (float) 1.0);
  conf.set(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES,
      ProportionalCapacityPreemptionPolicy.class.getCanonicalName());
  conf.setBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true);
  // FairScheduler doesn't support this test,
  // Set CapacityScheduler as the scheduler for this test.
  conf.set("yarn.resourcemanager.scheduler.class",
      CapacityScheduler.class.getName());

  mClock = mock(Clock.class);
  mCS = mock(CapacityScheduler.class);
  when(mCS.getResourceCalculator()).thenReturn(rc);
  lm = mock(RMNodeLabelsManager.class);
  schedConf = new CapacitySchedulerConfiguration();
  when(mCS.getConfiguration()).thenReturn(schedConf);
  rmContext = mock(RMContext.class);
  when(mCS.getRMContext()).thenReturn(rmContext);
  when(rmContext.getNodeLabelManager()).thenReturn(lm);
  mDisp = mock(EventHandler.class);
  rand = new Random();
  long seed = rand.nextLong();
  System.out.println(name.getMethodName() + " SEED: " + seed);
  rand.setSeed(seed);
  appAlloc = 0;
}
 

public ReduceTaskAttemptImpl(TaskId id, int attempt,
    EventHandler eventHandler, Path jobFile, int partition,
    int numMapTasks, JobConf conf,
    TaskAttemptListener taskAttemptListener,
    Token<JobTokenIdentifier> jobToken,
    Credentials credentials, Clock clock,
    AppContext appContext) {
  super(id, attempt, eventHandler, taskAttemptListener, jobFile, partition,
      conf, new String[] {}, jobToken, credentials, clock,
      appContext);
  this.numMapTasks = numMapTasks;
}
 

public MapTaskAttemptImpl(TaskId taskId, int attempt, 
    EventHandler eventHandler, Path jobFile, 
    int partition, TaskSplitMetaInfo splitInfo, JobConf conf,
    TaskAttemptListener taskAttemptListener, 
    Token<JobTokenIdentifier> jobToken,
    Credentials credentials, Clock clock,
    AppContext appContext) {
  super(taskId, attempt, eventHandler, 
      taskAttemptListener, jobFile, partition, conf, splitInfo.getLocations(),
      jobToken, credentials, clock, appContext);
  this.splitInfo = splitInfo;
}
 

public TaskHeartbeatHandler(EventHandler eventHandler, Clock clock,
    int numThreads) {
  super("TaskHeartbeatHandler");
  this.eventHandler = eventHandler;
  this.clock = clock;
  runningAttempts =
    new ConcurrentHashMap<TaskAttemptId, ReportTime>(16, 0.75f, numThreads);
}
 

public MockTaskAttemptImpl(TaskId taskId, int id, EventHandler eventHandler,
    TaskAttemptListener taskAttemptListener, Path jobFile, int partition,
    JobConf conf, Token<JobTokenIdentifier> jobToken,
    Credentials credentials, Clock clock,
    AppContext appContext, TaskType taskType) {
  super(taskId, id, eventHandler, taskAttemptListener, jobFile, partition, conf,
      dataLocations, jobToken, credentials, clock, appContext);
  this.taskType = taskType;
}
 

VertexWrapper(AppContext appContext, VertexPlan vertexPlan, Configuration conf,
              boolean checkVertexOnlyConf) {
  if (appContext == null) {
    mockAppContext = createDefaultMockAppContext();
    DAG mockDag = mock(DAG.class);
    doReturn(new Credentials()).when(mockDag).getCredentials();
    doReturn(mockDag).when(mockAppContext).getCurrentDAG();
  } else {
    mockAppContext = appContext;
  }

  Configuration dagConf = new Configuration(false);
  dagConf.set("abc1", "xyz1");
  dagConf.set("foo1", "bar1");

  this.vertexPlan = vertexPlan;

  vertex =
      new VertexImpl(TezVertexID.fromString("vertex_1418197758681_0001_1_00"), vertexPlan,
          "testvertex", conf, mock(EventHandler.class), mock(TaskCommunicatorManagerInterface.class),
          mock(Clock.class), mock(TaskHeartbeatHandler.class), false, mockAppContext,
          VertexLocationHint.create(new LinkedList<TaskLocationHint>()), null,
          new TaskSpecificLaunchCmdOption(conf), mock(StateChangeNotifier.class),
          dagConf);

  if (checkVertexOnlyConf) {
    Assert.assertEquals("xyz1", vertex.vertexOnlyConf.get("abc1"));
    Assert.assertEquals("bar2", vertex.vertexOnlyConf.get("foo1"));
    Assert.assertEquals("bar", vertex.vertexOnlyConf.get("foo"));
  }

}
 

public MapTaskImpl(JobId jobId, int partition, EventHandler eventHandler,
    Path remoteJobConfFile, JobConf conf,
    TaskSplitMetaInfo taskSplitMetaInfo,
    TaskAttemptListener taskAttemptListener,
    Token<JobTokenIdentifier> jobToken,
    Credentials credentials, Clock clock,
    int appAttemptId, MRAppMetrics metrics, AppContext appContext) {
  super(jobId, TaskType.MAP, partition, eventHandler, remoteJobConfFile,
      conf, taskAttemptListener, jobToken, credentials, clock,
      appAttemptId, metrics, appContext);
  this.taskSplitMetaInfo = taskSplitMetaInfo;
}
 

private void logStopped(int exitStatus) {
  final Clock clock = appContext.getClock();
  final HistoryEventHandler historyHandler = appContext.getHistoryHandler();
  ContainerStoppedEvent lEvt = new ContainerStoppedEvent(containerId,
      clock.getTime(), 
      exitStatus, 
      appContext.getApplicationAttemptId());
  historyHandler.handle(
      new DAGHistoryEvent(appContext.getCurrentDAGID(),lEvt));
}
 

public ReduceTaskAttemptImpl(TaskId id, int attempt,
    EventHandler eventHandler, Path jobFile, int partition,
    int numMapTasks, JobConf conf,
    TaskAttemptListener taskAttemptListener,
    Token<JobTokenIdentifier> jobToken,
    Credentials credentials, Clock clock,
    AppContext appContext) {
  super(id, attempt, eventHandler, taskAttemptListener, jobFile, partition,
      conf, new String[] {}, jobToken, credentials, clock,
      appContext);
  this.numMapTasks = numMapTasks;
}
 

public MyContainerAllocator(MyResourceManager rm, Configuration conf,
    ApplicationAttemptId appAttemptId, Job job, Clock clock) {
  super(createMockClientService(),
      createAppContext(appAttemptId, job, clock));
  this.rm = rm;
  super.init(conf);
  super.start();
}
 

public DAGAppMaster(ApplicationAttemptId applicationAttemptId,
    ContainerId containerId, String nmHost, int nmPort, int nmHttpPort,
    Clock clock, long appSubmitTime, boolean isSession, String workingDirectory,
    String [] localDirs, String[] logDirs, String clientVersion,
    Credentials credentials, String jobUserName, AMPluginDescriptorProto pluginDescriptorProto) {
  super(DAGAppMaster.class.getName());
  this.clock = clock;
  this.startTime = clock.getTime();
  this.appSubmitTime = appSubmitTime;
  this.appAttemptID = applicationAttemptId;
  this.containerID = containerId;
  this.nmHost = nmHost;
  this.nmPort = nmPort;
  this.nmHttpPort = nmHttpPort;
  this.state = DAGAppMasterState.NEW;
  this.isSession = isSession;
  this.workingDirectory = workingDirectory;
  this.localDirs = localDirs;
  this.logDirs = logDirs;
  this.shutdownHandler = createShutdownHandler();
  this.dagVersionInfo = new TezDagVersionInfo();
  this.clientVersion = clientVersion;
  this.amCredentials = credentials;
  this.amPluginDescriptorProto = pluginDescriptorProto;
  this.appMasterUgi = UserGroupInformation
      .createRemoteUser(jobUserName);
  this.appMasterUgi.addCredentials(amCredentials);

  this.containerLogs = getRunningLogURL(this.nmHost + ":" + this.nmHttpPort,
      this.containerID.toString(), this.appMasterUgi.getShortUserName());

  LOG.info("Created DAGAppMaster for application " + applicationAttemptId
      + ", versionInfo=" + dagVersionInfo.toString());
  TezCommonUtils.logCredentials(LOG, this.appMasterUgi.getCredentials(), "am");
}
 

private void logStopped(int exitStatus) {
  final Clock clock = appContext.getClock();
  final HistoryEventHandler historyHandler = appContext.getHistoryHandler();
  ContainerStoppedEvent lEvt = new ContainerStoppedEvent(containerId,
      clock.getTime(), 
      exitStatus, 
      appContext.getApplicationAttemptId());
  historyHandler.handle(
      new DAGHistoryEvent(appContext.getCurrentDAGID(),lEvt));
}
 

private void setupPlanFollower() throws Exception {
  ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
  mClock = mock(Clock.class);
  mAgent = mock(ReservationAgent.class);

  String reservationQ = testUtil.getFullReservationQueueName();
  CapacitySchedulerConfiguration csConf = cs.getConfiguration();
  csConf.setReservationWindow(reservationQ, 20L);
  csConf.setMaximumCapacity(reservationQ, 40);
  csConf.setAverageCapacity(reservationQ, 20);
  policy.init(reservationQ, csConf);
}
 

@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void testTimeout() throws InterruptedException {
  EventHandler mockHandler = mock(EventHandler.class);
  Clock clock = new SystemClock();
  TaskHeartbeatHandler hb = new TaskHeartbeatHandler(mockHandler, clock, 1);
  
  
  Configuration conf = new Configuration();
  conf.setInt(MRJobConfig.TASK_TIMEOUT, 10); //10 ms
  conf.setInt(MRJobConfig.TASK_TIMEOUT_CHECK_INTERVAL_MS, 10); //10 ms
  
  hb.init(conf);
  hb.start();
  try {
    ApplicationId appId = ApplicationId.newInstance(0l, 5);
    JobId jobId = MRBuilderUtils.newJobId(appId, 4);
    TaskId tid = MRBuilderUtils.newTaskId(jobId, 3, TaskType.MAP);
    TaskAttemptId taid = MRBuilderUtils.newTaskAttemptId(tid, 2);
    hb.register(taid);
    Thread.sleep(100);
    //Events only happen when the task is canceled
    verify(mockHandler, times(2)).handle(any(Event.class));
  } finally {
    hb.stop();
  }
}
 

private TaskAttemptImpl createMapTaskAttemptImplForTest(
    EventHandler eventHandler, TaskSplitMetaInfo taskSplitMetaInfo, Clock clock) {
  ApplicationId appId = ApplicationId.newInstance(1, 1);
  JobId jobId = MRBuilderUtils.newJobId(appId, 1);
  TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP);
  TaskAttemptListener taListener = mock(TaskAttemptListener.class);
  Path jobFile = mock(Path.class);
  JobConf jobConf = new JobConf();
  TaskAttemptImpl taImpl =
      new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile, 1,
          taskSplitMetaInfo, jobConf, taListener, null,
          null, clock, null);
  return taImpl;
}
 

public MockTaskImpl(JobId jobId, int partition,
    EventHandler eventHandler, Path remoteJobConfFile, JobConf conf,
    TaskAttemptListener taskAttemptListener,
    Token<JobTokenIdentifier> jobToken,
    Credentials credentials, Clock clock, int startCount,
    MRAppMetrics metrics, AppContext appContext, TaskType taskType) {
  super(jobId, taskType , partition, eventHandler,
      remoteJobConfFile, conf, taskAttemptListener,
      jobToken, credentials, clock,
      startCount, metrics, appContext);
  this.taskType = taskType;
}
 

MockTezClient(String name, TezConfiguration tezConf, boolean isSession,
    Map<String, LocalResource> localResources, Credentials credentials,
    Clock clock, AtomicBoolean mockAppLauncherGoFlag,
    boolean initFailFlag, boolean startFailFlag, int concurrency, int containers) {
  super(name, tezConf, isSession, localResources, credentials);
  this.client = new MockLocalClient(mockAppLauncherGoFlag, clock, initFailFlag, startFailFlag, 
      concurrency, containers);
}
 

public DatePartitionedLogger(Parser<T> parser, Path baseDir, Configuration conf, Clock clock)
    throws IOException {
  this.conf = new Configuration(conf);
  this.clock = clock;
  this.parser = parser;
  createDirIfNotExists(baseDir);
  this.basePath = baseDir.getFileSystem(conf).resolvePath(baseDir);
  FsPermission.setUMask(this.conf, FILE_UMASK);
}