org.apache.hadoop.yarn.webapp.WebApps Java Examples
The following examples show how to use
org.apache.hadoop.yarn.webapp.WebApps.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: HistoryClientService.java From big-c with Apache License 2.0 | 6 votes |
@VisibleForTesting protected void initializeWebApp(Configuration conf) { webApp = new HsWebApp(history); InetSocketAddress bindAddress = MRWebAppUtil.getJHSWebBindAddress(conf); // NOTE: there should be a .at(InetSocketAddress) WebApps .$for("jobhistory", HistoryClientService.class, this, "ws") .with(conf) .withHttpSpnegoKeytabKey( JHAdminConfig.MR_WEBAPP_SPNEGO_KEYTAB_FILE_KEY) .withHttpSpnegoPrincipalKey( JHAdminConfig.MR_WEBAPP_SPNEGO_USER_NAME_KEY) .at(NetUtils.getHostPortString(bindAddress)).start(webApp); String connectHost = MRWebAppUtil.getJHSWebappURLWithoutScheme(conf).split(":")[0]; MRWebAppUtil.setJHSWebappURLWithoutScheme(conf, connectHost + ":" + webApp.getListenerAddress().getPort()); }
Example #2
Source File: WebServer.java From big-c with Apache License 2.0 | 6 votes |
@Override protected void serviceStart() throws Exception { String bindAddress = WebAppUtils.getWebAppBindURL(getConfig(), YarnConfiguration.NM_BIND_HOST, WebAppUtils.getNMWebAppURLWithoutScheme(getConfig())); LOG.info("Instantiating NMWebApp at " + bindAddress); try { this.webApp = WebApps .$for("node", Context.class, this.nmContext, "ws") .at(bindAddress) .with(getConfig()) .withHttpSpnegoPrincipalKey( YarnConfiguration.NM_WEBAPP_SPNEGO_USER_NAME_KEY) .withHttpSpnegoKeytabKey( YarnConfiguration.NM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY) .start(this.nmWebApp); this.port = this.webApp.httpServer().getConnectorAddress(0).getPort(); } catch (Exception e) { String msg = "NMWebapps failed to start."; LOG.error(msg, e); throw new YarnRuntimeException(msg, e); } super.serviceStart(); }
Example #3
Source File: HistoryClientService.java From hadoop with Apache License 2.0 | 6 votes |
@VisibleForTesting protected void initializeWebApp(Configuration conf) { webApp = new HsWebApp(history); InetSocketAddress bindAddress = MRWebAppUtil.getJHSWebBindAddress(conf); // NOTE: there should be a .at(InetSocketAddress) WebApps .$for("jobhistory", HistoryClientService.class, this, "ws") .with(conf) .withHttpSpnegoKeytabKey( JHAdminConfig.MR_WEBAPP_SPNEGO_KEYTAB_FILE_KEY) .withHttpSpnegoPrincipalKey( JHAdminConfig.MR_WEBAPP_SPNEGO_USER_NAME_KEY) .at(NetUtils.getHostPortString(bindAddress)).start(webApp); String connectHost = MRWebAppUtil.getJHSWebappURLWithoutScheme(conf).split(":")[0]; MRWebAppUtil.setJHSWebappURLWithoutScheme(conf, connectHost + ":" + webApp.getListenerAddress().getPort()); }
Example #4
Source File: WebServer.java From hadoop with Apache License 2.0 | 6 votes |
@Override protected void serviceStart() throws Exception { String bindAddress = WebAppUtils.getWebAppBindURL(getConfig(), YarnConfiguration.NM_BIND_HOST, WebAppUtils.getNMWebAppURLWithoutScheme(getConfig())); LOG.info("Instantiating NMWebApp at " + bindAddress); try { this.webApp = WebApps .$for("node", Context.class, this.nmContext, "ws") .at(bindAddress) .with(getConfig()) .withHttpSpnegoPrincipalKey( YarnConfiguration.NM_WEBAPP_SPNEGO_USER_NAME_KEY) .withHttpSpnegoKeytabKey( YarnConfiguration.NM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY) .start(this.nmWebApp); this.port = this.webApp.httpServer().getConnectorAddress(0).getPort(); } catch (Exception e) { String msg = "NMWebapps failed to start."; LOG.error(msg, e); throw new YarnRuntimeException(msg, e); } super.serviceStart(); }
Example #5
Source File: WebUIService.java From tez with Apache License 2.0 | 5 votes |
@Override protected void serviceStart() throws Exception { if (tezAMWebApp != null) { // use AmIpFilter to restrict connections only from the rm proxy Configuration conf = getConfig(); conf.set("hadoop.http.filter.initializers", "org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer"); try { // Explicitly disabling SSL for the web service. For https we do not want AM users to allow // access to the keystore file for opening SSL listener. We can trust RM/NM to issue SSL // certificates, however AM user is not trusted. // ideally the withHttpPolicy should be used, however hadoop 2.2 does not have the api conf.set("yarn.http.policy", "HTTP_ONLY"); this.webApp = WebApps .$for(this.tezAMWebApp) .with(conf) .start(this.tezAMWebApp); InetSocketAddress address = webApp.getListenerAddress(); if (address != null) { InetSocketAddress bindAddress = NetUtils.createSocketAddrForHost( context.getAppMaster().getAppNMHost(), address.getPort()); String hostname = context.getAppMaster().getAppNMHost(); final int port = address.getPort(); if (bindAddress.getAddress() != null && bindAddress.getAddress().getCanonicalHostName() != null) { hostname = bindAddress.getAddress().getCanonicalHostName(); } else { LOG.warn("Failed to resolve canonical hostname for " + context.getAppMaster().getAppNMHost()); } trackingUrl = "http://" + hostname + ":" + port + "/ui/"; LOG.info("Instantiated WebUIService at " + trackingUrl); } } catch (Exception e) { LOG.error("Tez UI WebService failed to start.", e); throw new TezUncheckedException(e); } } super.serviceStart(); }
Example #6
Source File: MRClientService.java From big-c with Apache License 2.0 | 5 votes |
protected void serviceStart() throws Exception { Configuration conf = getConfig(); YarnRPC rpc = YarnRPC.create(conf); InetSocketAddress address = new InetSocketAddress(0); server = rpc.getServer(MRClientProtocol.class, protocolHandler, address, conf, appContext.getClientToAMTokenSecretManager(), conf.getInt(MRJobConfig.MR_AM_JOB_CLIENT_THREAD_COUNT, MRJobConfig.DEFAULT_MR_AM_JOB_CLIENT_THREAD_COUNT), MRJobConfig.MR_AM_JOB_CLIENT_PORT_RANGE); // Enable service authorization? if (conf.getBoolean( CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) { refreshServiceAcls(conf, new MRAMPolicyProvider()); } server.start(); this.bindAddress = NetUtils.createSocketAddrForHost(appContext.getNMHostname(), server.getListenerAddress().getPort()); LOG.info("Instantiated MRClientService at " + this.bindAddress); try { // Explicitly disabling SSL for map reduce task as we can't allow MR users // to gain access to keystore file for opening SSL listener. We can trust // RM/NM to issue SSL certificates but definitely not MR-AM as it is // running in user-land. webApp = WebApps.$for("mapreduce", AppContext.class, appContext, "ws") .withHttpPolicy(conf, Policy.HTTP_ONLY).start(new AMWebApp()); } catch (Exception e) { LOG.error("Webapps failed to start. Ignoring for now:", e); } super.serviceStart(); }
Example #7
Source File: TestRMWebApp.java From big-c with Apache License 2.0 | 5 votes |
public static void main(String[] args) throws Exception { // For manual testing WebApps.$for("yarn", new TestRMWebApp()).at(8888).inDevMode(). start(new RMWebApp(mockRm(2500, 8, 8, 8*GiB))).joinThread(); WebApps.$for("yarn", new TestRMWebApp()).at(8888).inDevMode(). start(new RMWebApp(mockFifoRm(10, 1, 4, 8*GiB))).joinThread(); }
Example #8
Source File: MRClientService.java From hadoop with Apache License 2.0 | 5 votes |
protected void serviceStart() throws Exception { Configuration conf = getConfig(); YarnRPC rpc = YarnRPC.create(conf); InetSocketAddress address = new InetSocketAddress(0); server = rpc.getServer(MRClientProtocol.class, protocolHandler, address, conf, appContext.getClientToAMTokenSecretManager(), conf.getInt(MRJobConfig.MR_AM_JOB_CLIENT_THREAD_COUNT, MRJobConfig.DEFAULT_MR_AM_JOB_CLIENT_THREAD_COUNT), MRJobConfig.MR_AM_JOB_CLIENT_PORT_RANGE); // Enable service authorization? if (conf.getBoolean( CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) { refreshServiceAcls(conf, new MRAMPolicyProvider()); } server.start(); this.bindAddress = NetUtils.createSocketAddrForHost(appContext.getNMHostname(), server.getListenerAddress().getPort()); LOG.info("Instantiated MRClientService at " + this.bindAddress); try { // Explicitly disabling SSL for map reduce task as we can't allow MR users // to gain access to keystore file for opening SSL listener. We can trust // RM/NM to issue SSL certificates but definitely not MR-AM as it is // running in user-land. webApp = WebApps.$for("mapreduce", AppContext.class, appContext, "ws") .withHttpPolicy(conf, Policy.HTTP_ONLY).start(new AMWebApp()); } catch (Exception e) { LOG.error("Webapps failed to start. Ignoring for now:", e); } super.serviceStart(); }
Example #9
Source File: TestRMWebApp.java From hadoop with Apache License 2.0 | 5 votes |
public static void main(String[] args) throws Exception { // For manual testing WebApps.$for("yarn", new TestRMWebApp()).at(8888).inDevMode(). start(new RMWebApp(mockRm(2500, 8, 8, 8*GiB))).joinThread(); WebApps.$for("yarn", new TestRMWebApp()).at(8888).inDevMode(). start(new RMWebApp(mockFifoRm(10, 1, 4, 8*GiB))).joinThread(); }
Example #10
Source File: InfoPage.java From XLearning with Apache License 2.0 | 5 votes |
@Override protected Class<? extends SubView> header() { try { if (WebApps.Builder.class.getMethod("build", WebApp.class) != null) { return HeaderBlock.class; } } catch (NoSuchMethodException e) { LOG.debug("current hadoop version don't have the method build of Class " + WebApps.class.toString() + ". For More Detail: " + e); return org.apache.hadoop.yarn.webapp.view.HeaderBlock.class; } return null; }
Example #11
Source File: HsJobPage.java From XLearning with Apache License 2.0 | 5 votes |
@Override protected Class<? extends SubView> header() { try { if (WebApps.Builder.class.getMethod("build", WebApp.class) != null) { return HeaderBlock.class; } } catch (NoSuchMethodException e) { LOG.debug("current hadoop version don't have the method build of Class " + WebApps.class.toString() + ". For More Detail: " + e); return org.apache.hadoop.yarn.webapp.view.HeaderBlock.class; } return null; }
Example #12
Source File: SCMWebServer.java From big-c with Apache License 2.0 | 4 votes |
@Override protected void serviceStart() throws Exception { SCMWebApp scmWebApp = new SCMWebApp(scm); this.webApp = WebApps.$for("sharedcache").at(bindAddress).start(scmWebApp); LOG.info("Instantiated " + SCMWebApp.class.getName() + " at " + bindAddress); }
Example #13
Source File: StreamingAppMasterService.java From Bats with Apache License 2.0 | 4 votes |
@Override protected void serviceStart() throws Exception { super.serviceStart(); if (UserGroupInformation.isSecurityEnabled()) { delegationTokenManager.startThreads(); } // write the connect address for containers to DFS InetSocketAddress connectAddress = NetUtils.getConnectAddress(this.heartbeatListener.getAddress()); URI connectUri = RecoverableRpcProxy.toConnectURI(connectAddress); FSRecoveryHandler recoveryHandler = new FSRecoveryHandler(dag.assertAppPath(), getConfig()); recoveryHandler.writeConnectUri(connectUri.toString()); // start web service try { org.mortbay.log.Log.setLog(null); } catch (Throwable throwable) { // SPOI-2687. As part of Pivotal Certification, we need to catch ClassNotFoundException as Pivotal was using // Jetty 7 where as other distros are using Jetty 6. // LOG.error("can't set the log to null: ", throwable); } try { Configuration config = SecurityUtils.configureWebAppSecurity(getConfig(), dag.getValue(Context.DAGContext.SSL_CONFIG)); WebApp webApp = WebApps.$for("stram", StramAppContext.class, appContext, "ws").with(config).start(new StramWebApp(this.dnmgr)); LOG.info("Started web service at port: " + webApp.port()); // best effort to produce FQDN for the client to connect with // (when SSL is enabled, it may be required to match the certificate) connectAddress = NetUtils.getConnectAddress(webApp.getListenerAddress()); String hostname = connectAddress.getAddress().getCanonicalHostName(); if (hostname.equals(connectAddress.getAddress().getHostAddress())) { // lookup didn't yield a name hostname = connectAddress.getHostName(); } appMasterTrackingUrl = hostname + ":" + webApp.port(); if (ConfigUtils.isSSLEnabled(config)) { appMasterTrackingUrl = "https://" + appMasterTrackingUrl; } LOG.info("Setting tracking URL to: " + appMasterTrackingUrl); } catch (Exception e) { LOG.error("Webapps failed to start. Ignoring for now:", e); } }
Example #14
Source File: StreamingAppMasterService.java From attic-apex-core with Apache License 2.0 | 4 votes |
@Override protected void serviceStart() throws Exception { super.serviceStart(); if (UserGroupInformation.isSecurityEnabled()) { delegationTokenManager.startThreads(); } // write the connect address for containers to DFS InetSocketAddress connectAddress = NetUtils.getConnectAddress(this.heartbeatListener.getAddress()); URI connectUri = RecoverableRpcProxy.toConnectURI(connectAddress); FSRecoveryHandler recoveryHandler = new FSRecoveryHandler(dag.assertAppPath(), getConfig()); recoveryHandler.writeConnectUri(connectUri.toString()); // start web service try { org.mortbay.log.Log.setLog(null); } catch (Throwable throwable) { // SPOI-2687. As part of Pivotal Certification, we need to catch ClassNotFoundException as Pivotal was using // Jetty 7 where as other distros are using Jetty 6. // LOG.error("can't set the log to null: ", throwable); } try { Configuration config = SecurityUtils.configureWebAppSecurity(getConfig(), dag.getValue(Context.DAGContext.SSL_CONFIG)); WebApp webApp = WebApps.$for("stram", StramAppContext.class, appContext, "ws").with(config).start(new StramWebApp(this.dnmgr)); LOG.info("Started web service at port: " + webApp.port()); // best effort to produce FQDN for the client to connect with // (when SSL is enabled, it may be required to match the certificate) connectAddress = NetUtils.getConnectAddress(webApp.getListenerAddress()); String hostname = connectAddress.getAddress().getCanonicalHostName(); if (hostname.equals(connectAddress.getAddress().getHostAddress())) { // lookup didn't yield a name hostname = connectAddress.getHostName(); } appMasterTrackingUrl = hostname + ":" + webApp.port(); if (ConfigUtils.isSSLEnabled(config)) { appMasterTrackingUrl = "https://" + appMasterTrackingUrl; } LOG.info("Setting tracking URL to: " + appMasterTrackingUrl); } catch (Exception e) { LOG.error("Webapps failed to start. Ignoring for now:", e); } }
Example #15
Source File: TestAMWebApp.java From big-c with Apache License 2.0 | 4 votes |
public static void main(String[] args) { WebApps.$for("yarn", AppContext.class, new MockAppContext(0, 8, 88, 4)). at(58888).inDevMode().start(new AMWebApp()).joinThread(); }
Example #16
Source File: ApplicationHistoryServer.java From big-c with Apache License 2.0 | 4 votes |
private void startWebApp() { Configuration conf = getConfig(); TimelineAuthenticationFilter.setTimelineDelegationTokenSecretManager( secretManagerService.getTimelineDelegationTokenSecretManager()); // Always load pseudo authentication filter to parse "user.name" in an URL // to identify a HTTP request's user in insecure mode. // When Kerberos authentication type is set (i.e., secure mode is turned on), // the customized filter will be loaded by the timeline server to do Kerberos // + DT authentication. String initializers = conf.get("hadoop.http.filter.initializers"); boolean modifiedInitializers = false; initializers = initializers == null || initializers.length() == 0 ? "" : initializers; if (!initializers.contains(CrossOriginFilterInitializer.class.getName())) { if(conf.getBoolean(YarnConfiguration .TIMELINE_SERVICE_HTTP_CROSS_ORIGIN_ENABLED, YarnConfiguration .TIMELINE_SERVICE_HTTP_CROSS_ORIGIN_ENABLED_DEFAULT)) { initializers = CrossOriginFilterInitializer.class.getName() + "," + initializers; modifiedInitializers = true; } } if (!initializers.contains(TimelineAuthenticationFilterInitializer.class .getName())) { initializers = TimelineAuthenticationFilterInitializer.class.getName() + "," + initializers; modifiedInitializers = true; } String[] parts = initializers.split(","); ArrayList<String> target = new ArrayList<String>(); for (String filterInitializer : parts) { filterInitializer = filterInitializer.trim(); if (filterInitializer.equals(AuthenticationFilterInitializer.class .getName())) { modifiedInitializers = true; continue; } target.add(filterInitializer); } String actualInitializers = org.apache.commons.lang.StringUtils.join(target, ","); if (modifiedInitializers) { conf.set("hadoop.http.filter.initializers", actualInitializers); } String bindAddress = WebAppUtils.getWebAppBindURL(conf, YarnConfiguration.TIMELINE_SERVICE_BIND_HOST, WebAppUtils.getAHSWebAppURLWithoutScheme(conf)); LOG.info("Instantiating AHSWebApp at " + bindAddress); try { webApp = WebApps .$for("applicationhistory", ApplicationHistoryClientService.class, ahsClientService, "ws") .with(conf).at(bindAddress).start( new AHSWebApp(timelineDataManager, ahsClientService)); } catch (Exception e) { String msg = "AHSWebApp failed to start."; LOG.error(msg, e); throw new YarnRuntimeException(msg, e); } }
Example #17
Source File: MyApp.java From hadoop with Apache License 2.0 | 4 votes |
public static void main(String[] args) throws Exception { WebApps.$for(new MyApp()).at(8888).inDevMode().start().joinThread(); }
Example #18
Source File: HelloWorld.java From hadoop with Apache License 2.0 | 4 votes |
public static void main(String[] args) { WebApps.$for(new HelloWorld()).at(8888).inDevMode().start().joinThread(); }
Example #19
Source File: SCMWebServer.java From hadoop with Apache License 2.0 | 4 votes |
@Override protected void serviceStart() throws Exception { SCMWebApp scmWebApp = new SCMWebApp(scm); this.webApp = WebApps.$for("sharedcache").at(bindAddress).start(scmWebApp); LOG.info("Instantiated " + SCMWebApp.class.getName() + " at " + bindAddress); }
Example #20
Source File: TestTwoColumnLayout.java From big-c with Apache License 2.0 | 4 votes |
public static void main(String[] args) { WebApps.$for("test").at(8888).inDevMode().start().joinThread(); }
Example #21
Source File: TestTwoColumnCssPage.java From big-c with Apache License 2.0 | 4 votes |
public static void main(String[] args) { WebApps.$for("test").at(8888).inDevMode().start().joinThread(); }
Example #22
Source File: HelloWorld.java From big-c with Apache License 2.0 | 4 votes |
public static void main(String[] args) { WebApps.$for(new HelloWorld()).at(8888).inDevMode().start().joinThread(); }
Example #23
Source File: MyApp.java From big-c with Apache License 2.0 | 4 votes |
public static void main(String[] args) throws Exception { WebApps.$for(new MyApp()).at(8888).inDevMode().start().joinThread(); }
Example #24
Source File: TestTwoColumnCssPage.java From hadoop with Apache License 2.0 | 4 votes |
public static void main(String[] args) { WebApps.$for("test").at(8888).inDevMode().start().joinThread(); }
Example #25
Source File: TestAMWebApp.java From hadoop with Apache License 2.0 | 4 votes |
public static void main(String[] args) { WebApps.$for("yarn", AppContext.class, new MockAppContext(0, 8, 88, 4)). at(58888).inDevMode().start(new AMWebApp()).joinThread(); }
Example #26
Source File: TestTwoColumnLayout.java From hadoop with Apache License 2.0 | 4 votes |
public static void main(String[] args) { WebApps.$for("test").at(8888).inDevMode().start().joinThread(); }
Example #27
Source File: ApplicationHistoryServer.java From hadoop with Apache License 2.0 | 4 votes |
private void startWebApp() { Configuration conf = getConfig(); TimelineAuthenticationFilter.setTimelineDelegationTokenSecretManager( secretManagerService.getTimelineDelegationTokenSecretManager()); // Always load pseudo authentication filter to parse "user.name" in an URL // to identify a HTTP request's user in insecure mode. // When Kerberos authentication type is set (i.e., secure mode is turned on), // the customized filter will be loaded by the timeline server to do Kerberos // + DT authentication. String initializers = conf.get("hadoop.http.filter.initializers"); boolean modifiedInitializers = false; initializers = initializers == null || initializers.length() == 0 ? "" : initializers; if (!initializers.contains(CrossOriginFilterInitializer.class.getName())) { if(conf.getBoolean(YarnConfiguration .TIMELINE_SERVICE_HTTP_CROSS_ORIGIN_ENABLED, YarnConfiguration .TIMELINE_SERVICE_HTTP_CROSS_ORIGIN_ENABLED_DEFAULT)) { initializers = CrossOriginFilterInitializer.class.getName() + "," + initializers; modifiedInitializers = true; } } if (!initializers.contains(TimelineAuthenticationFilterInitializer.class .getName())) { initializers = TimelineAuthenticationFilterInitializer.class.getName() + "," + initializers; modifiedInitializers = true; } String[] parts = initializers.split(","); ArrayList<String> target = new ArrayList<String>(); for (String filterInitializer : parts) { filterInitializer = filterInitializer.trim(); if (filterInitializer.equals(AuthenticationFilterInitializer.class .getName())) { modifiedInitializers = true; continue; } target.add(filterInitializer); } String actualInitializers = org.apache.commons.lang.StringUtils.join(target, ","); if (modifiedInitializers) { conf.set("hadoop.http.filter.initializers", actualInitializers); } String bindAddress = WebAppUtils.getWebAppBindURL(conf, YarnConfiguration.TIMELINE_SERVICE_BIND_HOST, WebAppUtils.getAHSWebAppURLWithoutScheme(conf)); LOG.info("Instantiating AHSWebApp at " + bindAddress); try { webApp = WebApps .$for("applicationhistory", ApplicationHistoryClientService.class, ahsClientService, "ws") .with(conf).at(bindAddress).start( new AHSWebApp(timelineDataManager, ahsClientService)); } catch (Exception e) { String msg = "AHSWebApp failed to start."; LOG.error(msg, e); throw new YarnRuntimeException(msg, e); } }