org.apache.hadoop.yarn.server.api.ApplicationInitializationContext Java Examples
The following examples show how to use
org.apache.hadoop.yarn.server.api.ApplicationInitializationContext.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ShuffleHandler.java From tez with Apache License 2.0 | 6 votes |
@Override public void initializeApplication(ApplicationInitializationContext context) { String user = context.getUser(); ApplicationId appId = context.getApplicationId(); ByteBuffer secret = context.getApplicationDataForService(); // TODO these bytes should be versioned try { Token<JobTokenIdentifier> jt = deserializeServiceData(secret); // TODO: Once SHuffle is out of NM, this can use MR APIs JobID jobId = new JobID(Long.toString(appId.getClusterTimestamp()), appId.getId()); recordJobShuffleInfo(jobId, user, jt); } catch (IOException e) { LOG.error("Error during initApp", e); // TODO add API to AuxiliaryServices to report failures } }
Example #2
Source File: ShuffleHandler.java From hadoop with Apache License 2.0 | 6 votes |
@Override public void initializeApplication(ApplicationInitializationContext context) { String user = context.getUser(); ApplicationId appId = context.getApplicationId(); ByteBuffer secret = context.getApplicationDataForService(); // TODO these bytes should be versioned try { Token<JobTokenIdentifier> jt = deserializeServiceData(secret); // TODO: Once SHuffle is out of NM, this can use MR APIs JobID jobId = new JobID(Long.toString(appId.getClusterTimestamp()), appId.getId()); recordJobShuffleInfo(jobId, user, jt); } catch (IOException e) { LOG.error("Error during initApp", e); // TODO add API to AuxiliaryServices to report failures } }
Example #3
Source File: ShuffleHandler.java From big-c with Apache License 2.0 | 6 votes |
@Override public void initializeApplication(ApplicationInitializationContext context) { String user = context.getUser(); ApplicationId appId = context.getApplicationId(); ByteBuffer secret = context.getApplicationDataForService(); // TODO these bytes should be versioned try { Token<JobTokenIdentifier> jt = deserializeServiceData(secret); // TODO: Once SHuffle is out of NM, this can use MR APIs JobID jobId = new JobID(Long.toString(appId.getClusterTimestamp()), appId.getId()); recordJobShuffleInfo(jobId, user, jt); } catch (IOException e) { LOG.error("Error during initApp", e); // TODO add API to AuxiliaryServices to report failures } }
Example #4
Source File: PullServerAuxService.java From incubator-tajo with Apache License 2.0 | 5 votes |
@Override public void initializeApplication(ApplicationInitializationContext appInitContext) { // TODO these bytes should be versioned // TODO: Once SHuffle is out of NM, this can use MR APIs this.appId = appInitContext.getApplicationId(); this.queryId = TajoIdUtils.parseQueryId(appId.toString()); this.userName = appInitContext.getUser(); userRsrc.put(this.appId.toString(), this.userName); }
Example #5
Source File: TajoPullServerService.java From tajo with Apache License 2.0 | 5 votes |
@Override public void initializeApplication(ApplicationInitializationContext context) { // TODO these bytes should be versioned // TODO: Once SHuffle is out of NM, this can use MR APIs String user = context.getUser(); ApplicationId appId = context.getApplicationId(); // ByteBuffer secret = context.getApplicationDataForService(); userRsrc.put(appId.toString(), user); }
Example #6
Source File: TestAuxServices.java From big-c with Apache License 2.0 | 5 votes |
@Override public void initializeApplication(ApplicationInitializationContext context) { ByteBuffer data = context.getApplicationDataForService(); assertEquals(idef, data.getChar()); assertEquals(expected_appId, data.getInt()); assertEquals(expected_appId, context.getApplicationId().getId()); }
Example #7
Source File: TestAuxServices.java From hadoop with Apache License 2.0 | 5 votes |
@Override public void initializeApplication(ApplicationInitializationContext context) { ByteBuffer data = context.getApplicationDataForService(); assertEquals(idef, data.getChar()); assertEquals(expected_appId, data.getInt()); assertEquals(expected_appId, context.getApplicationId().getId()); }
Example #8
Source File: TestShuffleProvider.java From big-c with Apache License 2.0 | 4 votes |
@Override public void initializeApplication(ApplicationInitializationContext context) { }
Example #9
Source File: TestShuffleHandler.java From tez with Apache License 2.0 | 4 votes |
@Test public void testRecoveryFromOtherVersions() throws IOException { final String user = "someuser"; final ApplicationId appId = ApplicationId.newInstance(12345, 1); final File tmpDir = new File(System.getProperty("test.build.data", System.getProperty("java.io.tmpdir")), TestShuffleHandler.class.getName()); Configuration conf = new Configuration(); conf.set(HADOOP_TMP_DIR, TEST_DIR.getAbsolutePath()); conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0); conf.setInt(ShuffleHandler.MAX_SHUFFLE_CONNECTIONS, 3); ShuffleHandler shuffle = new ShuffleHandler(); // emulate aux services startup with recovery enabled shuffle.setRecoveryPath(new Path(tmpDir.toString())); tmpDir.mkdirs(); try { shuffle.init(conf); shuffle.start(); // setup a shuffle token for an application DataOutputBuffer outputBuffer = new DataOutputBuffer(); outputBuffer.reset(); Token<JobTokenIdentifier> jt = new Token<JobTokenIdentifier>( "identifier".getBytes(), "password".getBytes(), new Text(user), new Text("shuffleService")); jt.write(outputBuffer); shuffle.initializeApplication(new ApplicationInitializationContext(user, appId, ByteBuffer.wrap(outputBuffer.getData(), 0, outputBuffer.getLength()))); // verify we are authorized to shuffle int rc = getShuffleResponseCode(shuffle, jt); Assert.assertEquals(HttpURLConnection.HTTP_OK, rc); // emulate shuffle handler restart shuffle.close(); shuffle = new ShuffleHandler(); shuffle.setRecoveryPath(new Path(tmpDir.toString())); shuffle.init(conf); shuffle.start(); // verify we are still authorized to shuffle to the old application rc = getShuffleResponseCode(shuffle, jt); Assert.assertEquals(HttpURLConnection.HTTP_OK, rc); Version version = Version.newInstance(1, 0); Assert.assertEquals(version, shuffle.getCurrentVersion()); // emulate shuffle handler restart with compatible version Version version11 = Version.newInstance(1, 1); // update version info before close shuffle shuffle.storeVersion(version11); Assert.assertEquals(version11, shuffle.loadVersion()); shuffle.close(); shuffle = new ShuffleHandler(); shuffle.setRecoveryPath(new Path(tmpDir.toString())); shuffle.init(conf); shuffle.start(); // shuffle version will be override by CURRENT_VERSION_INFO after restart // successfully. Assert.assertEquals(version, shuffle.loadVersion()); // verify we are still authorized to shuffle to the old application rc = getShuffleResponseCode(shuffle, jt); Assert.assertEquals(HttpURLConnection.HTTP_OK, rc); // emulate shuffle handler restart with incompatible version Version version21 = Version.newInstance(2, 1); shuffle.storeVersion(version21); Assert.assertEquals(version21, shuffle.loadVersion()); shuffle.close(); shuffle = new ShuffleHandler(); shuffle.setRecoveryPath(new Path(tmpDir.toString())); shuffle.init(conf); try { shuffle.start(); Assert.fail("Incompatible version, should expect fail here."); } catch (ServiceStateException e) { Assert.assertTrue("Exception message mismatch", e.getMessage().contains("Incompatible version for state DB schema:")); } } finally { if (shuffle != null) { shuffle.close(); } FileUtil.fullyDelete(tmpDir); } }
Example #10
Source File: TestShuffleHandler.java From tez with Apache License 2.0 | 4 votes |
@Test public void testRecovery() throws IOException { final String user = "someuser"; final ApplicationId appId = ApplicationId.newInstance(12345, 1); final JobID jobId = JobID.downgrade(TypeConverter.fromYarn(appId)); final File tmpDir = new File(System.getProperty("test.build.data", System.getProperty("java.io.tmpdir")), TestShuffleHandler.class.getName()); Configuration conf = new Configuration(); conf.set(HADOOP_TMP_DIR, TEST_DIR.getAbsolutePath()); conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0); conf.setInt(ShuffleHandler.MAX_SHUFFLE_CONNECTIONS, 3); ShuffleHandler shuffle = new ShuffleHandler(); // emulate aux services startup with recovery enabled shuffle.setRecoveryPath(new Path(tmpDir.toString())); tmpDir.mkdirs(); try { shuffle.init(conf); shuffle.start(); // setup a shuffle token for an application DataOutputBuffer outputBuffer = new DataOutputBuffer(); outputBuffer.reset(); Token<JobTokenIdentifier> jt = new Token<JobTokenIdentifier>( "identifier".getBytes(), "password".getBytes(), new Text(user), new Text("shuffleService")); jt.write(outputBuffer); shuffle.initializeApplication(new ApplicationInitializationContext(user, appId, ByteBuffer.wrap(outputBuffer.getData(), 0, outputBuffer.getLength()))); // verify we are authorized to shuffle int rc = getShuffleResponseCode(shuffle, jt); Assert.assertEquals(HttpURLConnection.HTTP_OK, rc); // emulate shuffle handler restart shuffle.close(); shuffle = new ShuffleHandler(); shuffle.setRecoveryPath(new Path(tmpDir.toString())); shuffle.init(conf); shuffle.start(); // verify we are still authorized to shuffle to the old application rc = getShuffleResponseCode(shuffle, jt); Assert.assertEquals(HttpURLConnection.HTTP_OK, rc); // shutdown app and verify access is lost shuffle.stopApplication(new ApplicationTerminationContext(appId)); rc = getShuffleResponseCode(shuffle, jt); Assert.assertEquals(HttpURLConnection.HTTP_UNAUTHORIZED, rc); // emulate shuffle handler restart shuffle.close(); shuffle = new ShuffleHandler(); shuffle.setRecoveryPath(new Path(tmpDir.toString())); shuffle.init(conf); shuffle.start(); // verify we still don't have access rc = getShuffleResponseCode(shuffle, jt); Assert.assertEquals(HttpURLConnection.HTTP_UNAUTHORIZED, rc); } finally { if (shuffle != null) { shuffle.close(); } FileUtil.fullyDelete(tmpDir); } }
Example #11
Source File: MyriadExecutorAuxService.java From incubator-myriad with Apache License 2.0 | 4 votes |
@Override public void initializeApplication(ApplicationInitializationContext initAppContext) { LOGGER.debug("initializeApplication"); }
Example #12
Source File: TestShuffleHandler.java From big-c with Apache License 2.0 | 4 votes |
@Test public void testRecoveryFromOtherVersions() throws IOException { final String user = "someuser"; final ApplicationId appId = ApplicationId.newInstance(12345, 1); final File tmpDir = new File(System.getProperty("test.build.data", System.getProperty("java.io.tmpdir")), TestShuffleHandler.class.getName()); Configuration conf = new Configuration(); conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0); conf.setInt(ShuffleHandler.MAX_SHUFFLE_CONNECTIONS, 3); ShuffleHandler shuffle = new ShuffleHandler(); // emulate aux services startup with recovery enabled shuffle.setRecoveryPath(new Path(tmpDir.toString())); tmpDir.mkdirs(); try { shuffle.init(conf); shuffle.start(); // setup a shuffle token for an application DataOutputBuffer outputBuffer = new DataOutputBuffer(); outputBuffer.reset(); Token<JobTokenIdentifier> jt = new Token<JobTokenIdentifier>( "identifier".getBytes(), "password".getBytes(), new Text(user), new Text("shuffleService")); jt.write(outputBuffer); shuffle.initializeApplication(new ApplicationInitializationContext(user, appId, ByteBuffer.wrap(outputBuffer.getData(), 0, outputBuffer.getLength()))); // verify we are authorized to shuffle int rc = getShuffleResponseCode(shuffle, jt); Assert.assertEquals(HttpURLConnection.HTTP_OK, rc); // emulate shuffle handler restart shuffle.close(); shuffle = new ShuffleHandler(); shuffle.setRecoveryPath(new Path(tmpDir.toString())); shuffle.init(conf); shuffle.start(); // verify we are still authorized to shuffle to the old application rc = getShuffleResponseCode(shuffle, jt); Assert.assertEquals(HttpURLConnection.HTTP_OK, rc); Version version = Version.newInstance(1, 0); Assert.assertEquals(version, shuffle.getCurrentVersion()); // emulate shuffle handler restart with compatible version Version version11 = Version.newInstance(1, 1); // update version info before close shuffle shuffle.storeVersion(version11); Assert.assertEquals(version11, shuffle.loadVersion()); shuffle.close(); shuffle = new ShuffleHandler(); shuffle.setRecoveryPath(new Path(tmpDir.toString())); shuffle.init(conf); shuffle.start(); // shuffle version will be override by CURRENT_VERSION_INFO after restart // successfully. Assert.assertEquals(version, shuffle.loadVersion()); // verify we are still authorized to shuffle to the old application rc = getShuffleResponseCode(shuffle, jt); Assert.assertEquals(HttpURLConnection.HTTP_OK, rc); // emulate shuffle handler restart with incompatible version Version version21 = Version.newInstance(2, 1); shuffle.storeVersion(version21); Assert.assertEquals(version21, shuffle.loadVersion()); shuffle.close(); shuffle = new ShuffleHandler(); shuffle.setRecoveryPath(new Path(tmpDir.toString())); shuffle.init(conf); try { shuffle.start(); Assert.fail("Incompatible version, should expect fail here."); } catch (ServiceStateException e) { Assert.assertTrue("Exception message mismatch", e.getMessage().contains("Incompatible version for state DB schema:")); } } finally { if (shuffle != null) { shuffle.close(); } FileUtil.fullyDelete(tmpDir); } }
Example #13
Source File: TestShuffleHandler.java From big-c with Apache License 2.0 | 4 votes |
@Test public void testRecovery() throws IOException { final String user = "someuser"; final ApplicationId appId = ApplicationId.newInstance(12345, 1); final JobID jobId = JobID.downgrade(TypeConverter.fromYarn(appId)); final File tmpDir = new File(System.getProperty("test.build.data", System.getProperty("java.io.tmpdir")), TestShuffleHandler.class.getName()); Configuration conf = new Configuration(); conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0); conf.setInt(ShuffleHandler.MAX_SHUFFLE_CONNECTIONS, 3); ShuffleHandler shuffle = new ShuffleHandler(); // emulate aux services startup with recovery enabled shuffle.setRecoveryPath(new Path(tmpDir.toString())); tmpDir.mkdirs(); try { shuffle.init(conf); shuffle.start(); // setup a shuffle token for an application DataOutputBuffer outputBuffer = new DataOutputBuffer(); outputBuffer.reset(); Token<JobTokenIdentifier> jt = new Token<JobTokenIdentifier>( "identifier".getBytes(), "password".getBytes(), new Text(user), new Text("shuffleService")); jt.write(outputBuffer); shuffle.initializeApplication(new ApplicationInitializationContext(user, appId, ByteBuffer.wrap(outputBuffer.getData(), 0, outputBuffer.getLength()))); // verify we are authorized to shuffle int rc = getShuffleResponseCode(shuffle, jt); Assert.assertEquals(HttpURLConnection.HTTP_OK, rc); // emulate shuffle handler restart shuffle.close(); shuffle = new ShuffleHandler(); shuffle.setRecoveryPath(new Path(tmpDir.toString())); shuffle.init(conf); shuffle.start(); // verify we are still authorized to shuffle to the old application rc = getShuffleResponseCode(shuffle, jt); Assert.assertEquals(HttpURLConnection.HTTP_OK, rc); // shutdown app and verify access is lost shuffle.stopApplication(new ApplicationTerminationContext(appId)); rc = getShuffleResponseCode(shuffle, jt); Assert.assertEquals(HttpURLConnection.HTTP_UNAUTHORIZED, rc); // emulate shuffle handler restart shuffle.close(); shuffle = new ShuffleHandler(); shuffle.setRecoveryPath(new Path(tmpDir.toString())); shuffle.init(conf); shuffle.start(); // verify we still don't have access rc = getShuffleResponseCode(shuffle, jt); Assert.assertEquals(HttpURLConnection.HTTP_UNAUTHORIZED, rc); } finally { if (shuffle != null) { shuffle.close(); } FileUtil.fullyDelete(tmpDir); } }
Example #14
Source File: TestShuffleProvider.java From big-c with Apache License 2.0 | 4 votes |
@Override public void initializeApplication(ApplicationInitializationContext context) { }
Example #15
Source File: TestAuxServices.java From big-c with Apache License 2.0 | 4 votes |
@Override public void initializeApplication( ApplicationInitializationContext initAppContext) { }
Example #16
Source File: TestShuffleHandler.java From hadoop with Apache License 2.0 | 4 votes |
@Test public void testRecoveryFromOtherVersions() throws IOException { final String user = "someuser"; final ApplicationId appId = ApplicationId.newInstance(12345, 1); final File tmpDir = new File(System.getProperty("test.build.data", System.getProperty("java.io.tmpdir")), TestShuffleHandler.class.getName()); Configuration conf = new Configuration(); conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0); conf.setInt(ShuffleHandler.MAX_SHUFFLE_CONNECTIONS, 3); ShuffleHandler shuffle = new ShuffleHandler(); // emulate aux services startup with recovery enabled shuffle.setRecoveryPath(new Path(tmpDir.toString())); tmpDir.mkdirs(); try { shuffle.init(conf); shuffle.start(); // setup a shuffle token for an application DataOutputBuffer outputBuffer = new DataOutputBuffer(); outputBuffer.reset(); Token<JobTokenIdentifier> jt = new Token<JobTokenIdentifier>( "identifier".getBytes(), "password".getBytes(), new Text(user), new Text("shuffleService")); jt.write(outputBuffer); shuffle.initializeApplication(new ApplicationInitializationContext(user, appId, ByteBuffer.wrap(outputBuffer.getData(), 0, outputBuffer.getLength()))); // verify we are authorized to shuffle int rc = getShuffleResponseCode(shuffle, jt); Assert.assertEquals(HttpURLConnection.HTTP_OK, rc); // emulate shuffle handler restart shuffle.close(); shuffle = new ShuffleHandler(); shuffle.setRecoveryPath(new Path(tmpDir.toString())); shuffle.init(conf); shuffle.start(); // verify we are still authorized to shuffle to the old application rc = getShuffleResponseCode(shuffle, jt); Assert.assertEquals(HttpURLConnection.HTTP_OK, rc); Version version = Version.newInstance(1, 0); Assert.assertEquals(version, shuffle.getCurrentVersion()); // emulate shuffle handler restart with compatible version Version version11 = Version.newInstance(1, 1); // update version info before close shuffle shuffle.storeVersion(version11); Assert.assertEquals(version11, shuffle.loadVersion()); shuffle.close(); shuffle = new ShuffleHandler(); shuffle.setRecoveryPath(new Path(tmpDir.toString())); shuffle.init(conf); shuffle.start(); // shuffle version will be override by CURRENT_VERSION_INFO after restart // successfully. Assert.assertEquals(version, shuffle.loadVersion()); // verify we are still authorized to shuffle to the old application rc = getShuffleResponseCode(shuffle, jt); Assert.assertEquals(HttpURLConnection.HTTP_OK, rc); // emulate shuffle handler restart with incompatible version Version version21 = Version.newInstance(2, 1); shuffle.storeVersion(version21); Assert.assertEquals(version21, shuffle.loadVersion()); shuffle.close(); shuffle = new ShuffleHandler(); shuffle.setRecoveryPath(new Path(tmpDir.toString())); shuffle.init(conf); try { shuffle.start(); Assert.fail("Incompatible version, should expect fail here."); } catch (ServiceStateException e) { Assert.assertTrue("Exception message mismatch", e.getMessage().contains("Incompatible version for state DB schema:")); } } finally { if (shuffle != null) { shuffle.close(); } FileUtil.fullyDelete(tmpDir); } }
Example #17
Source File: TestShuffleHandler.java From hadoop with Apache License 2.0 | 4 votes |
@Test public void testRecovery() throws IOException { final String user = "someuser"; final ApplicationId appId = ApplicationId.newInstance(12345, 1); final JobID jobId = JobID.downgrade(TypeConverter.fromYarn(appId)); final File tmpDir = new File(System.getProperty("test.build.data", System.getProperty("java.io.tmpdir")), TestShuffleHandler.class.getName()); Configuration conf = new Configuration(); conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0); conf.setInt(ShuffleHandler.MAX_SHUFFLE_CONNECTIONS, 3); ShuffleHandler shuffle = new ShuffleHandler(); // emulate aux services startup with recovery enabled shuffle.setRecoveryPath(new Path(tmpDir.toString())); tmpDir.mkdirs(); try { shuffle.init(conf); shuffle.start(); // setup a shuffle token for an application DataOutputBuffer outputBuffer = new DataOutputBuffer(); outputBuffer.reset(); Token<JobTokenIdentifier> jt = new Token<JobTokenIdentifier>( "identifier".getBytes(), "password".getBytes(), new Text(user), new Text("shuffleService")); jt.write(outputBuffer); shuffle.initializeApplication(new ApplicationInitializationContext(user, appId, ByteBuffer.wrap(outputBuffer.getData(), 0, outputBuffer.getLength()))); // verify we are authorized to shuffle int rc = getShuffleResponseCode(shuffle, jt); Assert.assertEquals(HttpURLConnection.HTTP_OK, rc); // emulate shuffle handler restart shuffle.close(); shuffle = new ShuffleHandler(); shuffle.setRecoveryPath(new Path(tmpDir.toString())); shuffle.init(conf); shuffle.start(); // verify we are still authorized to shuffle to the old application rc = getShuffleResponseCode(shuffle, jt); Assert.assertEquals(HttpURLConnection.HTTP_OK, rc); // shutdown app and verify access is lost shuffle.stopApplication(new ApplicationTerminationContext(appId)); rc = getShuffleResponseCode(shuffle, jt); Assert.assertEquals(HttpURLConnection.HTTP_UNAUTHORIZED, rc); // emulate shuffle handler restart shuffle.close(); shuffle = new ShuffleHandler(); shuffle.setRecoveryPath(new Path(tmpDir.toString())); shuffle.init(conf); shuffle.start(); // verify we still don't have access rc = getShuffleResponseCode(shuffle, jt); Assert.assertEquals(HttpURLConnection.HTTP_UNAUTHORIZED, rc); } finally { if (shuffle != null) { shuffle.close(); } FileUtil.fullyDelete(tmpDir); } }
Example #18
Source File: TestShuffleProvider.java From hadoop with Apache License 2.0 | 4 votes |
@Override public void initializeApplication(ApplicationInitializationContext context) { }
Example #19
Source File: TestShuffleProvider.java From hadoop with Apache License 2.0 | 4 votes |
@Override public void initializeApplication(ApplicationInitializationContext context) { }
Example #20
Source File: TestAuxServices.java From hadoop with Apache License 2.0 | 4 votes |
@Override public void initializeApplication( ApplicationInitializationContext initAppContext) { }