Java Code Examples for org.apache.hadoop.io.DataInputByteBuffer#reset()
The following examples show how to use
org.apache.hadoop.io.DataInputByteBuffer#reset() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: ContainerManagerImpl.java From big-c with Apache License 2.0 | 6 votes |
private Credentials parseCredentials(ContainerLaunchContext launchContext) throws IOException { Credentials credentials = new Credentials(); // //////////// Parse credentials ByteBuffer tokens = launchContext.getTokens(); if (tokens != null) { DataInputByteBuffer buf = new DataInputByteBuffer(); tokens.rewind(); buf.reset(tokens); credentials.readTokenStorageStream(buf); if (LOG.isDebugEnabled()) { for (Token<? extends TokenIdentifier> tk : credentials.getAllTokens()) { LOG.debug(tk.getService() + " = " + tk.toString()); } } } // //////////// End of parsing credentials return credentials; }
Example 2
Source File: ContainerManagerImpl.java From hadoop with Apache License 2.0 | 6 votes |
private Credentials parseCredentials(ContainerLaunchContext launchContext) throws IOException { Credentials credentials = new Credentials(); // //////////// Parse credentials ByteBuffer tokens = launchContext.getTokens(); if (tokens != null) { DataInputByteBuffer buf = new DataInputByteBuffer(); tokens.rewind(); buf.reset(tokens); credentials.readTokenStorageStream(buf); if (LOG.isDebugEnabled()) { for (Token<? extends TokenIdentifier> tk : credentials.getAllTokens()) { LOG.debug(tk.getService() + " = " + tk.toString()); } } } // //////////// End of parsing credentials return credentials; }
Example 3
Source File: YarnUtils.java From twill with Apache License 2.0 | 5 votes |
/** * Decodes {@link Credentials} from the given buffer. * If the buffer is null or empty, it returns an empty Credentials. */ public static Credentials decodeCredentials(ByteBuffer buffer) throws IOException { Credentials credentials = new Credentials(); if (buffer != null && buffer.hasRemaining()) { DataInputByteBuffer in = new DataInputByteBuffer(); in.reset(buffer); credentials.readTokenStorageStream(in); } return credentials; }
Example 4
Source File: TestTezClientUtils.java From tez with Apache License 2.0 | 5 votes |
@Test(timeout = 5000) public void testSessionTokenInAmClc() throws IOException, YarnException { TezConfiguration tezConf = new TezConfiguration(); tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, STAGING_DIR.getAbsolutePath()); ApplicationId appId = ApplicationId.newInstance(1000, 1); DAG dag = DAG.create("testdag"); dag.addVertex(Vertex.create("testVertex", ProcessorDescriptor.create("processorClassname"), 1) .setTaskLaunchCmdOpts("initialLaunchOpts")); Credentials credentials = new Credentials(); JobTokenSecretManager jobTokenSecretManager = new JobTokenSecretManager(); TezClientUtils.createSessionToken(appId.toString(), jobTokenSecretManager, credentials); Token<JobTokenIdentifier> jobToken = TokenCache.getSessionToken(credentials); assertNotNull(jobToken); AMConfiguration amConf = new AMConfiguration(tezConf, new HashMap<String, LocalResource>(), credentials); ApplicationSubmissionContext appSubmissionContext = TezClientUtils.createApplicationSubmissionContext(appId, dag, "amName", amConf, new HashMap<String, LocalResource>(), credentials, false, new TezApiVersionInfo(), null, null); ContainerLaunchContext amClc = appSubmissionContext.getAMContainerSpec(); Map<String, ByteBuffer> amServiceData = amClc.getServiceData(); assertNotNull(amServiceData); assertEquals(1, amServiceData.size()); DataInputByteBuffer dibb = new DataInputByteBuffer(); dibb.reset(amServiceData.values().iterator().next()); Token<JobTokenIdentifier> jtSent = new Token<JobTokenIdentifier>(); jtSent.readFields(dibb); assertTrue(Arrays.equals(jobToken.getIdentifier(), jtSent.getIdentifier())); }
Example 5
Source File: ShuffleHandler.java From tez with Apache License 2.0 | 5 votes |
/** * A helper function to deserialize the metadata returned by ShuffleHandler. * @param meta the metadata returned by the ShuffleHandler * @return the port the Shuffle Handler is listening on to serve shuffle data. */ public static int deserializeMetaData(ByteBuffer meta) throws IOException { //TODO this should be returning a class not just an int DataInputByteBuffer in = new DataInputByteBuffer(); in.reset(meta); int port = in.readInt(); return port; }
Example 6
Source File: RMAppManager.java From big-c with Apache License 2.0 | 5 votes |
protected Credentials parseCredentials( ApplicationSubmissionContext application) throws IOException { Credentials credentials = new Credentials(); DataInputByteBuffer dibb = new DataInputByteBuffer(); ByteBuffer tokens = application.getAMContainerSpec().getTokens(); if (tokens != null) { dibb.reset(tokens); credentials.readTokenStorageStream(dibb); tokens.rewind(); } return credentials; }
Example 7
Source File: RMAppImpl.java From big-c with Apache License 2.0 | 5 votes |
protected Credentials parseCredentials() throws IOException { Credentials credentials = new Credentials(); DataInputByteBuffer dibb = new DataInputByteBuffer(); ByteBuffer tokens = submissionContext.getAMContainerSpec().getTokens(); if (tokens != null) { dibb.reset(tokens); credentials.readTokenStorageStream(dibb); tokens.rewind(); } return credentials; }
Example 8
Source File: AMLauncher.java From big-c with Apache License 2.0 | 5 votes |
private void setupTokens( ContainerLaunchContext container, ContainerId containerID) throws IOException { Map<String, String> environment = container.getEnvironment(); environment.put(ApplicationConstants.APPLICATION_WEB_PROXY_BASE_ENV, application.getWebProxyBase()); // Set AppSubmitTime and MaxAppAttempts to be consumable by the AM. ApplicationId applicationId = application.getAppAttemptId().getApplicationId(); environment.put( ApplicationConstants.APP_SUBMIT_TIME_ENV, String.valueOf(rmContext.getRMApps() .get(applicationId) .getSubmitTime())); environment.put(ApplicationConstants.MAX_APP_ATTEMPTS_ENV, String.valueOf(rmContext.getRMApps().get( applicationId).getMaxAppAttempts())); Credentials credentials = new Credentials(); DataInputByteBuffer dibb = new DataInputByteBuffer(); if (container.getTokens() != null) { // TODO: Don't do this kind of checks everywhere. dibb.reset(container.getTokens()); credentials.readTokenStorageStream(dibb); } // Add AMRMToken Token<AMRMTokenIdentifier> amrmToken = createAndSetAMRMToken(); if (amrmToken != null) { credentials.addToken(amrmToken.getService(), amrmToken); } DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); container.setTokens(ByteBuffer.wrap(dob.getData(), 0, dob.getLength())); }
Example 9
Source File: ShuffleHandler.java From hadoop with Apache License 2.0 | 5 votes |
static Token<JobTokenIdentifier> deserializeServiceData(ByteBuffer secret) throws IOException { DataInputByteBuffer in = new DataInputByteBuffer(); in.reset(secret); Token<JobTokenIdentifier> jt = new Token<JobTokenIdentifier>(); jt.readFields(in); return jt; }
Example 10
Source File: TezRuntimeUtils.java From tez with Apache License 2.0 | 5 votes |
public static int deserializeShuffleProviderMetaData(ByteBuffer meta) throws IOException { DataInputByteBuffer in = new DataInputByteBuffer(); try { in.reset(meta); int port = in.readInt(); return port; } finally { in.close(); } }
Example 11
Source File: TajoPullServerService.java From incubator-tajo with Apache License 2.0 | 5 votes |
/** * A helper function to deserialize the metadata returned by PullServerAuxService. * @param meta the metadata returned by the PullServerAuxService * @return the port the PullServer Handler is listening on to serve shuffle data. */ public static int deserializeMetaData(ByteBuffer meta) throws IOException { //TODO this should be returning a class not just an int DataInputByteBuffer in = new DataInputByteBuffer(); in.reset(meta); return in.readInt(); }
Example 12
Source File: DagTypeConverters.java From tez with Apache License 2.0 | 5 votes |
public static Credentials convertByteStringToCredentials(ByteString byteString) { if (byteString == null) { return null; } DataInputByteBuffer dib = new DataInputByteBuffer(); dib.reset(byteString.asReadOnlyByteBuffer()); Credentials credentials = new Credentials(); try { credentials.readTokenStorageStream(dib); return credentials; } catch (IOException e) { throw new TezUncheckedException("Failed to deserialize Credentials", e); } }
Example 13
Source File: TestAMAuthorization.java From hadoop with Apache License 2.0 | 5 votes |
public Credentials getContainerCredentials() throws IOException { Credentials credentials = new Credentials(); DataInputByteBuffer buf = new DataInputByteBuffer(); containerTokens.rewind(); buf.reset(containerTokens); credentials.readTokenStorageStream(buf); return credentials; }
Example 14
Source File: PullServerAuxService.java From incubator-tajo with Apache License 2.0 | 5 votes |
/** * A helper function to deserialize the metadata returned by PullServerAuxService. * @param meta the metadata returned by the PullServerAuxService * @return the port the PullServer Handler is listening on to serve shuffle data. */ public static int deserializeMetaData(ByteBuffer meta) throws IOException { //TODO this should be returning a class not just an int DataInputByteBuffer in = new DataInputByteBuffer(); in.reset(meta); return in.readInt(); }
Example 15
Source File: RMAppManager.java From hadoop with Apache License 2.0 | 5 votes |
protected Credentials parseCredentials( ApplicationSubmissionContext application) throws IOException { Credentials credentials = new Credentials(); DataInputByteBuffer dibb = new DataInputByteBuffer(); ByteBuffer tokens = application.getAMContainerSpec().getTokens(); if (tokens != null) { dibb.reset(tokens); credentials.readTokenStorageStream(dibb); tokens.rewind(); } return credentials; }
Example 16
Source File: RMAppImpl.java From hadoop with Apache License 2.0 | 5 votes |
protected Credentials parseCredentials() throws IOException { Credentials credentials = new Credentials(); DataInputByteBuffer dibb = new DataInputByteBuffer(); ByteBuffer tokens = submissionContext.getAMContainerSpec().getTokens(); if (tokens != null) { dibb.reset(tokens); credentials.readTokenStorageStream(dibb); tokens.rewind(); } return credentials; }
Example 17
Source File: ShuffleHandler.java From tez with Apache License 2.0 | 5 votes |
static Token<JobTokenIdentifier> deserializeServiceData(ByteBuffer secret) throws IOException { DataInputByteBuffer in = new DataInputByteBuffer(); in.reset(secret); Token<JobTokenIdentifier> jt = new Token<JobTokenIdentifier>(); jt.readFields(in); return jt; }
Example 18
Source File: TestDelegationTokenRenewer.java From big-c with Apache License 2.0 | 4 votes |
@Test public void testAppSubmissionWithoutDelegationToken() throws Exception { conf.setBoolean(YarnConfiguration.RM_PROXY_USER_PRIVILEGES_ENABLED, true); // create token2 Text userText2 = new Text("user2"); DelegationTokenIdentifier dtId2 = new DelegationTokenIdentifier(new Text("user2"), new Text("renewer2"), userText2); final Token<DelegationTokenIdentifier> token2 = new Token<DelegationTokenIdentifier>(dtId2.getBytes(), "password2".getBytes(), dtId2.getKind(), new Text("service2")); final MockRM rm = new TestSecurityMockRM(conf, null) { @Override protected DelegationTokenRenewer createDelegationTokenRenewer() { return new DelegationTokenRenewer() { @Override protected Token<?>[] obtainSystemTokensForUser(String user, final Credentials credentials) throws IOException { credentials.addToken(token2.getService(), token2); return new Token<?>[] { token2 }; } }; } }; rm.start(); // submit an app without delegationToken RMApp app = rm.submitApp(200); // wait for the new retrieved hdfs token. GenericTestUtils.waitFor(new Supplier<Boolean>() { public Boolean get() { return rm.getRMContext().getDelegationTokenRenewer() .getDelegationTokens().contains(token2); } }, 1000, 20000); // check nm can retrieve the token final MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService()); nm1.registerNode(); NodeHeartbeatResponse response = nm1.nodeHeartbeat(true); ByteBuffer tokenBuffer = response.getSystemCredentialsForApps().get(app.getApplicationId()); Assert.assertNotNull(tokenBuffer); Credentials appCredentials = new Credentials(); DataInputByteBuffer buf = new DataInputByteBuffer(); tokenBuffer.rewind(); buf.reset(tokenBuffer); appCredentials.readTokenStorageStream(buf); Assert.assertTrue(appCredentials.getAllTokens().contains(token2)); }
Example 19
Source File: TestCommit.java From tez with Apache License 2.0 | 4 votes |
public CountingOutputCommitterConfig(UserPayload payload) throws IOException { DataInputByteBuffer in = new DataInputByteBuffer(); in.reset(payload.getPayload()); this.readFields(in); }
Example 20
Source File: TestTaskAttemptContainerRequest.java From big-c with Apache License 2.0 | 4 votes |
@Test public void testAttemptContainerRequest() throws Exception { final Text SECRET_KEY_ALIAS = new Text("secretkeyalias"); final byte[] SECRET_KEY = ("secretkey").getBytes(); Map<ApplicationAccessType, String> acls = new HashMap<ApplicationAccessType, String>(1); acls.put(ApplicationAccessType.VIEW_APP, "otheruser"); ApplicationId appId = ApplicationId.newInstance(1, 1); JobId jobId = MRBuilderUtils.newJobId(appId, 1); TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP); Path jobFile = mock(Path.class); EventHandler eventHandler = mock(EventHandler.class); TaskAttemptListener taListener = mock(TaskAttemptListener.class); when(taListener.getAddress()).thenReturn(new InetSocketAddress("localhost", 0)); JobConf jobConf = new JobConf(); jobConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class); jobConf.setBoolean("fs.file.impl.disable.cache", true); jobConf.set(JobConf.MAPRED_MAP_TASK_ENV, ""); // setup UGI for security so tokens and keys are preserved jobConf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); UserGroupInformation.setConfiguration(jobConf); Credentials credentials = new Credentials(); credentials.addSecretKey(SECRET_KEY_ALIAS, SECRET_KEY); Token<JobTokenIdentifier> jobToken = new Token<JobTokenIdentifier>( ("tokenid").getBytes(), ("tokenpw").getBytes(), new Text("tokenkind"), new Text("tokenservice")); TaskAttemptImpl taImpl = new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile, 1, mock(TaskSplitMetaInfo.class), jobConf, taListener, jobToken, credentials, new SystemClock(), null); jobConf.set(MRJobConfig.APPLICATION_ATTEMPT_ID, taImpl.getID().toString()); ContainerLaunchContext launchCtx = TaskAttemptImpl.createContainerLaunchContext(acls, jobConf, jobToken, taImpl.createRemoteTask(), TypeConverter.fromYarn(jobId), mock(WrappedJvmID.class), taListener, credentials); Assert.assertEquals("ACLs mismatch", acls, launchCtx.getApplicationACLs()); Credentials launchCredentials = new Credentials(); DataInputByteBuffer dibb = new DataInputByteBuffer(); dibb.reset(launchCtx.getTokens()); launchCredentials.readTokenStorageStream(dibb); // verify all tokens specified for the task attempt are in the launch context for (Token<? extends TokenIdentifier> token : credentials.getAllTokens()) { Token<? extends TokenIdentifier> launchToken = launchCredentials.getToken(token.getService()); Assert.assertNotNull("Token " + token.getService() + " is missing", launchToken); Assert.assertEquals("Token " + token.getService() + " mismatch", token, launchToken); } // verify the secret key is in the launch context Assert.assertNotNull("Secret key missing", launchCredentials.getSecretKey(SECRET_KEY_ALIAS)); Assert.assertTrue("Secret key mismatch", Arrays.equals(SECRET_KEY, launchCredentials.getSecretKey(SECRET_KEY_ALIAS))); }