Java Code Examples for org.apache.tez.common.TezUtils#createConfFromUserPayload()
The following examples show how to use
org.apache.tez.common.TezUtils#createConfFromUserPayload() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TezTestServiceContainerLauncher.java From tez with Apache License 2.0 | 6 votes |
public TezTestServiceContainerLauncher(ContainerLauncherContext containerLauncherContext) { super(containerLauncherContext); try { conf = TezUtils.createConfFromUserPayload(getContext().getInitialUserPayload()); } catch (IOException e) { throw new RuntimeException(e); } int numThreads = conf.getInt( TezTestServiceConfConstants.TEZ_TEST_SERVICE_AM_COMMUNICATOR_NUM_THREADS, TezTestServiceConfConstants.TEZ_TEST_SERVICE_AM_COMMUNICATOR_NUM_THREADS_DEFAULT); this.servicePort = conf.getInt( TezTestServiceConfConstants.TEZ_TEST_SERVICE_RPC_PORT, -1); Preconditions.checkArgument(servicePort > 0, TezTestServiceConfConstants.TEZ_TEST_SERVICE_RPC_PORT + " must be set"); this.communicator = new TezTestServiceCommunicator(numThreads); this.tokenIdentifier = getContext().getApplicationAttemptId().getApplicationId().toString(); this.appAttemptId = getContext().getApplicationAttemptId(); }
Example 2
Source File: YarnTaskSchedulerService.java From tez with Apache License 2.0 | 6 votes |
@Private @VisibleForTesting YarnTaskSchedulerService(TaskSchedulerContext taskSchedulerContext, TezAMRMClientAsync<CookieContainerRequest> client) { super(taskSchedulerContext); this.containerSignatureMatcher = taskSchedulerContext.getContainerSignatureMatcher(); this.amRmClient = client; this.appHostName = taskSchedulerContext.getAppHostName(); this.appHostPort = taskSchedulerContext.getAppClientPort(); this.appTrackingUrl = taskSchedulerContext.getAppTrackingUrl(); try { this.conf = TezUtils.createConfFromUserPayload(taskSchedulerContext.getInitialUserPayload()); } catch (IOException e) { throw new TezUncheckedException( "Failed to deserialize payload for " + YarnTaskSchedulerService.class.getSimpleName(), e); } }
Example 3
Source File: OnFileSortedOutput.java From incubator-tez with Apache License 2.0 | 6 votes |
@Override public synchronized List<Event> initialize() throws IOException { this.startTime = System.nanoTime(); this.conf = TezUtils.createConfFromUserPayload(getContext().getUserPayload()); // Initializing this parametr in this conf since it is used in multiple // places (wherever LocalDirAllocator is used) - TezTaskOutputFiles, // TezMerger, etc. this.conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS, getContext().getWorkDirs()); this.memoryUpdateCallbackHandler = new MemoryUpdateCallbackHandler(); getContext().requestInitialMemory( ExternalSorter.getInitialMemoryRequirement(conf, getContext().getTotalMemoryAvailableToTask()), memoryUpdateCallbackHandler); sendEmptyPartitionDetails = this.conf.getBoolean( TezJobConfig.TEZ_RUNTIME_EMPTY_PARTITION_INFO_VIA_EVENTS_ENABLED, TezJobConfig.TEZ_RUNTIME_EMPTY_PARTITION_INFO_VIA_EVENTS_ENABLED_DEFAULT); return Collections.emptyList(); }
Example 4
Source File: OnFileUnorderedKVOutput.java From incubator-tez with Apache License 2.0 | 6 votes |
@Override public synchronized List<Event> initialize() throws Exception { this.conf = TezUtils.createConfFromUserPayload(getContext() .getUserPayload()); this.conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS, getContext().getWorkDirs()); getContext().requestInitialMemory(0l, null); // mandatory call this.dataViaEventsEnabled = conf.getBoolean( TezJobConfig.TEZ_RUNTIME_BROADCAST_DATA_VIA_EVENTS_ENABLED, TezJobConfig.TEZ_RUNTIME_BROADCAST_DATA_VIA_EVENTS_ENABLED_DEFAULT); this.dataViaEventsMaxSize = conf.getInt( TezJobConfig.TEZ_RUNTIME_BROADCAST_DATA_VIA_EVENTS_MAX_SIZE, TezJobConfig.TEZ_RUNTIME_BROADCAST_DATA_VIA_EVENTS_MAX_SIZE_DEFAULT); LOG.info(this.getClass().getSimpleName() + " running with params -> " + "dataViaEventsEnabled: " + dataViaEventsEnabled + ", dataViaEventsMaxSize: " + dataViaEventsMaxSize); this.kvWriter = new FileBasedKVWriter(getContext(), conf); return Collections.emptyList(); }
Example 5
Source File: TezContainerLauncherImpl.java From tez with Apache License 2.0 | 6 votes |
public TezContainerLauncherImpl(ContainerLauncherContext containerLauncherContext) { super(containerLauncherContext); try { this.conf = TezUtils.createConfFromUserPayload(containerLauncherContext.getInitialUserPayload()); } catch (IOException e) { throw new TezUncheckedException( "Failed to parse user payload for " + TezContainerLauncherImpl.class.getSimpleName(), e); } conf.setInt( CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY, 0); this.limitOnPoolSize = conf.getInt( TezConfiguration.TEZ_AM_CONTAINERLAUNCHER_THREAD_COUNT_LIMIT, TezConfiguration.TEZ_AM_CONTAINERLAUNCHER_THREAD_COUNT_LIMIT_DEFAULT); LOG.info("Upper limit on the thread pool size is " + this.limitOnPoolSize); }
Example 6
Source File: TestAMRecovery.java From tez with Apache License 2.0 | 5 votes |
@Override public void initialize() { super.initialize(); try { conf = TezUtils.createConfFromUserPayload(getContext().getUserPayload()); } catch (IOException e) { e.printStackTrace(); } }
Example 7
Source File: MRCombiner.java From tez with Apache License 2.0 | 5 votes |
public MRCombiner(TaskContext taskContext) throws IOException { final Configuration userConf = TezUtils.createConfFromUserPayload(taskContext.getUserPayload()); useNewApi = ConfigUtils.useNewApi(userConf); if (useNewApi) { conf = new JobConf(userConf); } else { conf = userConf; } assert(taskContext instanceof InputContext || taskContext instanceof OutputContext); if (taskContext instanceof OutputContext) { this.keyClass = ConfigUtils.getIntermediateOutputKeyClass(conf); this.valClass = ConfigUtils.getIntermediateOutputValueClass(conf); this.comparator = ConfigUtils.getIntermediateOutputKeyComparator(conf); this.reporter = new MRTaskReporter((OutputContext)taskContext); } else { this.keyClass = ConfigUtils.getIntermediateInputKeyClass(conf); this.valClass = ConfigUtils.getIntermediateInputValueClass(conf); this.comparator = ConfigUtils.getIntermediateInputKeyComparator(conf); this.reporter = new MRTaskReporter((InputContext)taskContext); } combineInputRecordsCounter = taskContext.getCounters().findCounter(TaskCounter.COMBINE_INPUT_RECORDS); combineOutputRecordsCounter = taskContext.getCounters().findCounter(TaskCounter.COMBINE_OUTPUT_RECORDS); boolean isMap = conf.getBoolean(MRConfig.IS_MAP_PROCESSOR,false); this.mrTaskAttemptID = new TaskAttemptID( new TaskID(String.valueOf(taskContext.getApplicationId() .getClusterTimestamp()), taskContext.getApplicationId().getId(), isMap ? TaskType.MAP : TaskType.REDUCE, taskContext.getTaskIndex()), taskContext.getTaskAttemptNumber()); LOG.info("Using combineKeyClass: " + keyClass + ", combineValueClass: " + valClass + ", combineComparator: " +comparator + ", useNewApi: " + useNewApi); }
Example 8
Source File: YarnTaskSchedulerService.java From tez with Apache License 2.0 | 5 votes |
public YarnTaskSchedulerService(TaskSchedulerContext taskSchedulerContext) { super(taskSchedulerContext); this.containerSignatureMatcher = taskSchedulerContext.getContainerSignatureMatcher(); this.amRmClient = TezAMRMClientAsync.createAMRMClientAsync(1000, this); this.appHostName = taskSchedulerContext.getAppHostName(); this.appHostPort = taskSchedulerContext.getAppClientPort(); this.appTrackingUrl = taskSchedulerContext.getAppTrackingUrl(); try { this.conf = TezUtils.createConfFromUserPayload(taskSchedulerContext.getInitialUserPayload()); } catch (IOException e) { throw new TezUncheckedException( "Failed to deserialize payload for " + YarnTaskSchedulerService.class.getSimpleName(), e); } }
Example 9
Source File: MRCombiner.java From incubator-tez with Apache License 2.0 | 5 votes |
public MRCombiner(TezTaskContext taskContext) throws IOException { this.conf = TezUtils.createConfFromUserPayload(taskContext.getUserPayload()); assert(taskContext instanceof TezInputContext || taskContext instanceof TezOutputContext); if (taskContext instanceof TezOutputContext) { this.keyClass = ConfigUtils.getIntermediateOutputKeyClass(conf); this.valClass = ConfigUtils.getIntermediateOutputValueClass(conf); this.comparator = ConfigUtils.getIntermediateOutputKeyComparator(conf); this.reporter = new MRTaskReporter((TezOutputContext)taskContext); } else { this.keyClass = ConfigUtils.getIntermediateInputKeyClass(conf); this.valClass = ConfigUtils.getIntermediateInputValueClass(conf); this.comparator = ConfigUtils.getIntermediateInputKeyComparator(conf); this.reporter = new MRTaskReporter((TezInputContext)taskContext); } this.useNewApi = ConfigUtils.useNewApi(conf); combineInputKeyCounter = taskContext.getCounters().findCounter(TaskCounter.COMBINE_INPUT_RECORDS); combineInputValueCounter = taskContext.getCounters().findCounter(TaskCounter.COMBINE_OUTPUT_RECORDS); boolean isMap = conf.getBoolean(MRConfig.IS_MAP_PROCESSOR,false); this.mrTaskAttemptID = new TaskAttemptID( new TaskID(String.valueOf(taskContext.getApplicationId() .getClusterTimestamp()), taskContext.getApplicationId().getId(), isMap ? TaskType.MAP : TaskType.REDUCE, taskContext.getTaskIndex()), taskContext.getTaskAttemptNumber()); LOG.info("Using combineKeyClass: " + keyClass + ", combineValueClass: " + valClass + ", combineComparator: " +comparator + ", useNewApi: " + useNewApi); }
Example 10
Source File: TestEntityDescriptor.java From tez with Apache License 2.0 | 5 votes |
public void verifyResults(InputDescriptor entityDescriptor, InputDescriptor deserialized, UserPayload payload, String confVal) throws IOException { Assert.assertEquals(entityDescriptor.getClassName(), deserialized.getClassName()); // History text is not serialized when sending to tasks Assert.assertNull(deserialized.getHistoryText()); Assert.assertArrayEquals(payload.deepCopyAsArray(), deserialized.getUserPayload().deepCopyAsArray()); Configuration deserializedConf = TezUtils.createConfFromUserPayload(deserialized.getUserPayload()); Assert.assertEquals(confVal, deserializedConf.get("testKey")); }
Example 11
Source File: UnorderedPartitionedKVOutputConfig.java From tez with Apache License 2.0 | 5 votes |
@InterfaceAudience.Private public void fromUserPayload(UserPayload payload) { try { this.conf = TezUtils.createConfFromUserPayload(payload); } catch (IOException e) { throw new RuntimeException(e); } }
Example 12
Source File: LocalMergedInput.java From incubator-tez with Apache License 2.0 | 5 votes |
@Override public List<Event> initialize() throws IOException { getContext().requestInitialMemory(0l, null); // mandatory call. getContext().inputIsReady(); this.conf = TezUtils.createConfFromUserPayload(getContext().getUserPayload()); if (getNumPhysicalInputs() == 0) { return Collections.emptyList(); } LocalShuffle localShuffle = new LocalShuffle(getContext(), conf, getNumPhysicalInputs()); rawIter = localShuffle.run(); createValuesIterator(); return Collections.emptyList(); }
Example 13
Source File: TestAMRecovery.java From tez with Apache License 2.0 | 5 votes |
@Override public void initialize() { super.initialize(); try { conf = TezUtils.createConfFromUserPayload(getContext().getUserPayload()); } catch (IOException e) { e.printStackTrace(); } }
Example 14
Source File: TestAMRecovery.java From tez with Apache License 2.0 | 5 votes |
@Override public void initialize() { super.initialize(); try { conf = TezUtils.createConfFromUserPayload(getContext().getUserPayload()); } catch (IOException e) { e.printStackTrace(); } }
Example 15
Source File: OnFileUnorderedPartitionedKVOutput.java From incubator-tez with Apache License 2.0 | 5 votes |
@Override public synchronized List<Event> initialize(TezOutputContext outputContext) throws Exception { this.outputContext = outputContext; this.conf = TezUtils.createConfFromUserPayload(outputContext.getUserPayload()); this.conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS, outputContext.getWorkDirs()); this.conf.setInt(TezRuntimeFrameworkConfigs.TEZ_RUNTIME_NUM_EXPECTED_PARTITIONS, this.numPhysicalOutputs); this.memoryUpdateCallbackHandler = new MemoryUpdateCallbackHandler(); outputContext.requestInitialMemory( UnorderedPartitionedKVWriter.getInitialMemoryRequirement(conf, outputContext.getTotalMemoryAvailableToTask()), memoryUpdateCallbackHandler); return Collections.emptyList(); }
Example 16
Source File: TezDAGStats.java From spork with Apache License 2.0 | 4 votes |
/** * Updates the statistics after a DAG is finished. */ public void accumulateStats(TezJob tezJob) throws IOException { //For Oozie Pig action job id matching compatibility with MR mode appId = tezJob.getApplicationId().toString().replace("application", "job"); setConf(tezJob.getConfiguration()); DAG dag = tezJob.getDAG(); hdfsBytesRead = -1; hdfsBytesWritten = -1; TezCounters tezCounters = tezJob.getDAGCounters(); if (tezCounters != null) { counters = covertToHadoopCounters(tezCounters); CounterGroup dagGrp = tezCounters.getGroup(DAG_COUNTER_GROUP); totalTasks = (int) dagGrp.findCounter("TOTAL_LAUNCHED_TASKS").getValue(); CounterGroup fsGrp = tezCounters.getGroup(FS_COUNTER_GROUP); fileBytesRead = fsGrp.findCounter("FILE_BYTES_READ").getValue(); fileBytesWritten = fsGrp.findCounter("FILE_BYTES_WRITTEN").getValue(); hdfsBytesRead = fsGrp.findCounter("HDFS_BYTES_READ").getValue(); hdfsBytesWritten = fsGrp.findCounter("HDFS_BYTES_WRITTEN").getValue(); } else { LOG.warn("Failed to get counters for DAG: " + dag.getName()); } for (Entry<String, TezVertexStats> entry : tezVertexStatsMap.entrySet()) { Vertex v = dag.getVertex(entry.getKey()); if (v != null && tezVertexStatsMap.containsKey(v.getName())) { TezVertexStats vertexStats = entry.getValue(); UserPayload payload = v.getProcessorDescriptor().getUserPayload(); Configuration conf = TezUtils.createConfFromUserPayload(payload); vertexStats.setConf(conf); VertexStatus status = tezJob.getVertexStatus(v.getName()); if (status == null) { LOG.warn("Failed to get status for vertex: " + v.getName()); continue; } vertexStats.accumulateStats(status, v.getParallelism()); if(vertexStats.getInputs() != null && !vertexStats.getInputs().isEmpty()) { inputs.addAll(vertexStats.getInputs()); } if(vertexStats.getOutputs() != null && !vertexStats.getOutputs().isEmpty()) { outputs.addAll(vertexStats.getOutputs()); } /*if (vertexStats.getHdfsBytesRead() >= 0) { hdfsBytesRead = (hdfsBytesRead == -1) ? 0 : hdfsBytesRead; hdfsBytesRead += vertexStats.getHdfsBytesRead(); } if (vertexStats.getHdfsBytesWritten() >= 0) { hdfsBytesWritten = (hdfsBytesWritten == -1) ? 0 : hdfsBytesWritten; hdfsBytesWritten += vertexStats.getHdfsBytesWritten(); }*/ if (!vertexStats.getMultiStoreCounters().isEmpty()) { multiStoreCounters.putAll(vertexStats.getMultiStoreCounters()); } numberMaps += vertexStats.getNumberMaps(); numberReduces += vertexStats.getNumberReduces(); mapInputRecords += vertexStats.getMapInputRecords(); mapOutputRecords += vertexStats.getMapOutputRecords(); reduceInputRecords += vertexStats.getReduceInputRecords(); reduceOutputRecords += vertexStats.getReduceOutputRecords(); spillCount += vertexStats.getSMMSpillCount(); activeSpillCountObj += vertexStats.getProactiveSpillCountObjects(); activeSpillCountRecs += vertexStats.getProactiveSpillCountRecs(); } } }
Example 17
Source File: RootInputVertexManager.java From tez with Apache License 2.0 | 4 votes |
@Override public void initialize() { UserPayload userPayload = getContext().getUserPayload(); if (userPayload == null || userPayload.getPayload() == null || userPayload.getPayload().limit() == 0) { throw new RuntimeException("Could not initialize RootInputVertexManager" + " from provided user payload"); } try { conf = TezUtils.createConfFromUserPayload(userPayload); } catch (IOException e) { throw new RuntimeException("Could not initialize RootInputVertexManager" + " from provided user payload", e); } slowStartEnabled = conf.getBoolean( TEZ_ROOT_INPUT_VERTEX_MANAGER_ENABLE_SLOW_START, TEZ_ROOT_INPUT_VERTEX_MANAGER_ENABLE_SLOW_START_DEFAULT); if(slowStartEnabled) { slowStartMinFraction = conf.getFloat( TEZ_ROOT_INPUT_VERTEX_MANAGER_MIN_SRC_FRACTION, TEZ_ROOT_INPUT_VERTEX_MANAGER_MIN_SRC_FRACTION_DEFAULT); slowStartMaxFraction = conf.getFloat( TEZ_ROOT_INPUT_VERTEX_MANAGER_MAX_SRC_FRACTION, Math.max(slowStartMinFraction, TEZ_ROOT_INPUT_VERTEX_MANAGER_MAX_SRC_FRACTION_DEFAULT)); } else { slowStartMinFraction = 0; slowStartMaxFraction = 0; } if (slowStartMinFraction < 0 || slowStartMaxFraction > 1 || slowStartMaxFraction < slowStartMinFraction) { throw new IllegalArgumentException( "Invalid values for slowStartMinFraction" + "/slowStartMaxFraction. Min " + "cannot be < 0, max cannot be > 1, and max cannot be < min." + ", configuredMin=" + slowStartMinFraction + ", configuredMax=" + slowStartMaxFraction); } updatePendingTasks(); }
Example 18
Source File: MRHelpers.java From incubator-tez with Apache License 2.0 | 4 votes |
@LimitedPrivate("Hive, Pig") @Unstable public static Configuration createConfFromUserPayload(byte[] bb) throws IOException { return TezUtils.createConfFromUserPayload(bb); }
Example 19
Source File: TestTaskSchedulerManager.java From tez with Apache License 2.0 | 4 votes |
@Test(timeout = 5000) public void testCustomTaskSchedulerSetup() throws IOException { Configuration conf = new Configuration(false); conf.set("testkey", "testval"); UserPayload defaultPayload = TezUtils.createUserPayloadFromConf(conf); String customSchedulerName = "fakeScheduler"; List<NamedEntityDescriptor> taskSchedulers = new LinkedList<>(); ByteBuffer bb = ByteBuffer.allocate(4); bb.putInt(0, 3); UserPayload userPayload = UserPayload.create(bb); taskSchedulers.add( new NamedEntityDescriptor(customSchedulerName, FakeTaskScheduler.class.getName()) .setUserPayload(userPayload)); taskSchedulers.add(new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null) .setUserPayload(defaultPayload)); TSEHForMultipleSchedulersTest tseh = new TSEHForMultipleSchedulersTest(mockAppContext, mockClientService, mockEventHandler, mockSigMatcher, mockWebUIService, taskSchedulers, false); tseh.init(conf); tseh.start(); // Verify that the YARN task scheduler is installed by default assertTrue(tseh.getYarnSchedulerCreated()); assertFalse(tseh.getUberSchedulerCreated()); assertEquals(2, tseh.getNumCreateInvocations()); // Verify the order of the schedulers assertEquals(customSchedulerName, tseh.getTaskSchedulerName(0)); assertEquals(TezConstants.getTezYarnServicePluginName(), tseh.getTaskSchedulerName(1)); // Verify the payload setup for the custom task scheduler assertNotNull(tseh.getTaskSchedulerContext(0)); assertEquals(bb, tseh.getTaskSchedulerContext(0).getInitialUserPayload().getPayload()); // Verify the payload on the yarn scheduler assertNotNull(tseh.getTaskSchedulerContext(1)); Configuration parsed = TezUtils.createConfFromUserPayload(tseh.getTaskSchedulerContext(1).getInitialUserPayload()); assertEquals("testval", parsed.get("testkey")); }
Example 20
Source File: PigProcessor.java From spork with Apache License 2.0 | 4 votes |
@SuppressWarnings("unchecked") @Override public void initialize() throws Exception { // Reset any static variables to avoid conflict in container-reuse. sampleVertex = null; sampleMap = null; // Reset static variables cleared for avoiding OOM. new JVMReuseImpl().cleanupStaticData(); UserPayload payload = getContext().getUserPayload(); conf = TezUtils.createConfFromUserPayload(payload); PigContext.setPackageImportList((ArrayList<String>) ObjectSerializer .deserialize(conf.get("udf.import.list"))); PigContext pc = (PigContext) ObjectSerializer.deserialize(conf.get("pig.pigContext")); // To determine front-end in UDFContext conf.set(MRConfiguration.JOB_APPLICATION_ATTEMPT_ID, getContext().getUniqueIdentifier()); conf.set(PigConstants.TASK_INDEX, Integer.toString(getContext().getTaskIndex())); UDFContext.getUDFContext().addJobConf(conf); UDFContext.getUDFContext().deserialize(); String execPlanString = conf.get(PLAN); execPlan = (PhysicalPlan) ObjectSerializer.deserialize(execPlanString); SchemaTupleBackend.initialize(conf, pc); PigMapReduce.sJobContext = HadoopShims.createJobContext(conf, new org.apache.hadoop.mapreduce.JobID()); // Set the job conf as a thread-local member of PigMapReduce // for backwards compatibility with the existing code base. PigMapReduce.sJobConfInternal.set(conf); Utils.setDefaultTimeZone(conf); boolean aggregateWarning = "true".equalsIgnoreCase(pc.getProperties().getProperty("aggregate.warning")); PigStatusReporter pigStatusReporter = PigStatusReporter.getInstance(); pigStatusReporter.setContext(new TezTaskContext(getContext())); pigHadoopLogger = PigHadoopLogger.getInstance(); pigHadoopLogger.setReporter(pigStatusReporter); pigHadoopLogger.setAggregate(aggregateWarning); PhysicalOperator.setPigLogger(pigHadoopLogger); LinkedList<TezTaskConfigurable> tezTCs = PlanHelper.getPhysicalOperators(execPlan, TezTaskConfigurable.class); for (TezTaskConfigurable tezTC : tezTCs){ tezTC.initialize(getContext()); } }