Java Code Examples for org.apache.http.HttpHost#getHostName()

/**
 * Creates a mock {@link IncomingRequest}.
 * 
 * @param uri
 *            the uri
 * @return the {@link IncomingRequest}
 */
public static IncomingRequest.Builder createIncomingRequest(String uri) {
    HttpHost httpHost = UriUtils.extractHost(uri);
    String scheme = httpHost.getSchemeName();
    String host = httpHost.getHostName();
    int port = httpHost.getPort();
    RequestLine requestLine = new BasicRequestLine("GET", uri, HttpVersion.HTTP_1_1);
    IncomingRequest.Builder builder = IncomingRequest.builder(requestLine);
    builder.setContext(new ContainerRequestContext() {
    });
    // Remove default ports
    if (port == -1 || (port == Http.DEFAULT_HTTP_PORT && "http".equals(scheme))
            || (port == Http.DEFAULT_HTTPS_PORT && "https".equals(scheme))) {
        builder.addHeader("Host", host);
    } else {
        builder.addHeader("Host", host + ":" + port);
    }
    builder.setSession(new MockSession());
    return builder;
}
 

public static Tuple<Inet4Address, Inet6Address> getRandomAddressesFromHost(final HttpHost host) throws HttpException {
  final List<InetAddress> ipList;
  try {
    ipList = Arrays.asList(InetAddress.getAllByName(host.getHostName()));
  } catch (final UnknownHostException e) {
    throw new HttpException("Could not resolve " + host.getHostName(), e);
  }
  final List<Inet6Address> ip6 = new ArrayList<>();
  final List<Inet4Address> ip4 = new ArrayList<>();

  Collections.reverse(ipList);
  for (final InetAddress ip : ipList) {
    if (ip instanceof Inet6Address)
      ip6.add((Inet6Address) ip);
    else if (ip instanceof Inet4Address)
      ip4.add((Inet4Address) ip);
  }
  return new Tuple<>(getRandomFromList(ip4), getRandomFromList(ip6));
}
 

private static void updateProxySetting(HttpConfig config, DefaultHttpClient httpClient) {
	if (config == null || httpClient == null){
		return;
	}

	HttpHost[] proxyChain = null;

	if (globalProxy != null) {
		//全局代理设置
		proxyChain = new HttpHost[] {globalProxy};
		if (globalProxyCredentials != null){
			AuthScope globalProxyAuthScope = new AuthScope(globalProxy.getHostName(), globalProxy.getPort());
			httpClient.getCredentialsProvider().setCredentials(globalProxyAuthScope, globalProxyCredentials);
		}
	}

	if (config.isUseProxy() && StringUtil.isNotEmpty(config.getHttpProxyHost())){
		HttpHost spProxy = new HttpHost(config.getHttpProxyHost(), config.getHttpProxyPort());
		if (globalProxy != null){
			proxyChain = new HttpHost[] {globalProxy, spProxy};
		} else {
			proxyChain = new HttpHost[] {spProxy};
		}
		if (StringUtil.isNotEmpty(config.getHttpProxyUser())){
			AuthScope spProxyAuthScope = new AuthScope(spProxy.getHostName(), spProxy.getPort());
			UsernamePasswordCredentials spProxyCredentials = new UsernamePasswordCredentials(
					config.getHttpProxyUser(), config.getHttpProxyPassword());
			httpClient.getCredentialsProvider().setCredentials(spProxyAuthScope, spProxyCredentials);
		}
	}

	httpClient.setRoutePlanner(new LibHttpRoutePlanner(connectionManager.getSchemeRegistry(), proxyChain));
}
 

/**
 * Define the Credentials
 *
 * @param httpClientBuilder
 * @param url
 * @return {@link HttpClientBuilder}
 */
private HttpClientBuilder configCredentials(HttpClientBuilder httpClientBuilder, final String url) {

	final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
	for (final Map.Entry<HttpHost, UsernamePasswordCredentials> entry : authenticationMap.entrySet()) {

		final HttpHost httpHost = entry.getKey();
		final UsernamePasswordCredentials usernamePasswordCredentials = entry.getValue();
		final AuthScope authscope = new AuthScope(httpHost.getHostName(), httpHost.getPort());
		credentialsProvider.setCredentials(authscope, usernamePasswordCredentials);
	}
	httpClientBuilder = httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
	httpClientBuilder = configureProxy(httpClientBuilder, credentialsProvider, url);
	return httpClientBuilder;
}
 

public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException {
    AuthState authState = (AuthState) context.getAttribute(ClientContext.TARGET_AUTH_STATE);
    CredentialsProvider credsProvider = (CredentialsProvider) context.getAttribute(
            ClientContext.CREDS_PROVIDER);
    HttpHost targetHost = (HttpHost) context.getAttribute(ExecutionContext.HTTP_TARGET_HOST);

    if (authState.getAuthScheme() == null) {
        AuthScope authScope = new AuthScope(targetHost.getHostName(), targetHost.getPort());
        Credentials creds = credsProvider.getCredentials(authScope);
        if (creds != null) {
            authState.setAuthScheme(new BasicScheme());
            authState.setCredentials(creds);
        }
    }
}
 

@Override
public Socket connectSocket(
        int connectTimeout,
        Socket socket,
        HttpHost host,
        InetSocketAddress remoteAddress,
        InetSocketAddress localAddress,
        HttpContext context) throws IOException {
    String hostName = host.getHostName();
    int port = remoteAddress.getPort();
    InetSocketAddress unresolvedRemote = InetSocketAddress.createUnresolved(hostName, port);
    return super.connectSocket(connectTimeout, socket, host, unresolvedRemote, localAddress, context);
}
 

public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException {
    AuthState authState = (AuthState) context.getAttribute(ClientContext.TARGET_AUTH_STATE);
    CredentialsProvider credsProvider = (CredentialsProvider) context.getAttribute(
            ClientContext.CREDS_PROVIDER);
    HttpHost targetHost = (HttpHost) context.getAttribute(ExecutionContext.HTTP_TARGET_HOST);

    if (authState.getAuthScheme() == null) {
        AuthScope authScope = new AuthScope(targetHost.getHostName(), targetHost.getPort());
        Credentials creds = credsProvider.getCredentials(authScope);
        if (creds != null) {
            authState.setAuthScheme(new BasicScheme());
            authState.setCredentials(creds);
        }
    }
}
 

@Override
public Socket connectSocket(
        int connectTimeout,
        Socket socket,
        HttpHost host,
        InetSocketAddress remoteAddress,
        InetSocketAddress localAddress,
        HttpContext context) throws IOException {
    String hostName = host.getHostName();
    int port = remoteAddress.getPort();
    InetSocketAddress unresolvedRemote = InetSocketAddress.createUnresolved(hostName, port);
    return super.connectSocket(connectTimeout, socket, host, unresolvedRemote, localAddress, context);
}
 

@Override
public Socket connectSocket(
        int connectTimeout,
        Socket socket,
        HttpHost host,
        InetSocketAddress remoteAddress,
        InetSocketAddress localAddress,
        HttpContext context) throws IOException {
    String hostName = host.getHostName();
    int port = remoteAddress.getPort();
    InetSocketAddress unresolvedRemote = InetSocketAddress.createUnresolved(hostName, port);
    return super.connectSocket(connectTimeout, socket, host, unresolvedRemote, localAddress, context);
}
 

@Override
protected NameIntValuePair<String> getHost(Object[] args) {
    final Object arg = args[HTTP_HOST_INDEX];
    if (arg instanceof HttpHost) {
        final HttpHost httpHost = (HttpHost) arg;
        return new NameIntValuePair<String>(httpHost.getHostName(), httpHost.getPort());
    }
    return null;
}
 

@Override
public HttpRoute determineRoute(final HttpHost host, final HttpRequest request, final HttpContext context) throws HttpException {
  Args.notNull(request, "Request");
  if (host == null) {
    throw new ProtocolException("Target host is not specified");
  }
  final HttpClientContext clientContext = HttpClientContext.adapt(context);
  final RequestConfig config = clientContext.getRequestConfig();
  int remotePort;
  if (host.getPort() <= 0) {
    try {
      remotePort = schemePortResolver.resolve(host);
    } catch (final UnsupportedSchemeException e) {
      throw new HttpException(e.getMessage());
    }
  } else
    remotePort = host.getPort();

  final Tuple<Inet4Address, Inet6Address> remoteAddresses = IpAddressTools.getRandomAddressesFromHost(host);
  final Tuple<InetAddress, InetAddress> addresses = determineAddressPair(remoteAddresses);

  final HttpHost target = new HttpHost(addresses.r, host.getHostName(), remotePort, host.getSchemeName());
  final HttpHost proxy = config.getProxy();
  final boolean secure = target.getSchemeName().equalsIgnoreCase("https");
  clientContext.setAttribute(CHOSEN_IP_ATTRIBUTE, addresses.l);
  log.debug("Setting route context attribute to {}", addresses.l);
  if (proxy == null) {
    return new HttpRoute(target, addresses.l, secure);
  } else {
    return new HttpRoute(target, addresses.l, proxy, secure);
  }
}
 

public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException {
  AuthState authState = (AuthState) context.getAttribute(ClientContext.TARGET_AUTH_STATE);
  CredentialsProvider credsProvider = (CredentialsProvider) context.getAttribute(ClientContext.CREDS_PROVIDER);
  HttpHost targetHost = (HttpHost) context.getAttribute(ExecutionContext.HTTP_TARGET_HOST);

  if (authState.getAuthScheme() == null) {
    AuthScope authScope = new AuthScope(targetHost.getHostName(), targetHost.getPort());
    Credentials creds = credsProvider.getCredentials(authScope);
    if (creds != null) {
      authState.setAuthScheme(new BasicScheme());
      authState.setCredentials(creds);
    }
  }
}
 

protected HttpHost getKey(final HttpHost host) {
    if (host.getPort() <= 0) {
        final int port;
        try {
            port = schemePortResolver.resolve(host);
        } catch (final UnsupportedSchemeException ignore) {
            return host;
        }
        return new HttpHost(host.getHostName(), port, host.getSchemeName());
    } else {
        return host;
    }
}
 

public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException {
    AuthState authState = (AuthState) context.getAttribute(ClientContext.TARGET_AUTH_STATE);
    CredentialsProvider credsProvider = (CredentialsProvider) context.getAttribute(
            ClientContext.CREDS_PROVIDER);
    HttpHost targetHost = (HttpHost) context.getAttribute(ExecutionContext.HTTP_TARGET_HOST);

    if (authState.getAuthScheme() == null) {
        AuthScope authScope = new AuthScope(targetHost.getHostName(), targetHost.getPort());
        Credentials creds = credsProvider.getCredentials(authScope);
        if (creds != null) {
            authState.setAuthScheme(new BasicScheme());
            authState.setCredentials(creds);
        }
    }
}
 

protected void build() throws URISyntaxException {

        URI uri = MDSInterface2.getRoot(this);
        mHost = new HttpHost(uri.getHost(), uri.getPort(), uri.getScheme());

        CredentialsProvider credsProvider = new BasicCredentialsProvider();
        AuthScope authScope = new AuthScope(mHost.getHostName(), mHost.getPort());
        Credentials creds = new UsernamePasswordCredentials("username", "password");
        credsProvider.setCredentials(authScope, creds);

        mClient = new DefaultHttpClient();
        ((AbstractHttpClient) mClient).getCredentialsProvider().setCredentials(
                authScope, creds);
    }
 

@Override
public HttpRequest generateRequest() throws IOException, HttpException {
    // first read the volatile, span will become visible as a result
    HttpRequest request = delegate.generateRequest();

    if (request != null) {
        RequestLine requestLine = request.getRequestLine();
        if (requestLine != null) {
            String method = requestLine.getMethod();
            span.withName(method).appendToName(" ");
            span.getContext().getHttp().withMethod(method).withUrl(requestLine.getUri());
        }
        span.propagateTraceContext(request, headerSetter);
    }

    HttpHost host = getTarget();
    //noinspection ConstantConditions
    if (host != null) {
        String hostname = host.getHostName();
        if (hostname != null) {
            span.appendToName(hostname);
            HttpClientHelper.setDestinationServiceDetails(span, host.getSchemeName(), hostname, host.getPort());
        }
    }

    //noinspection ConstantConditions
    return request;
}
 

private void updateAuthState(
        final AuthState authState,
        final HttpHost host,
        final CredentialsProvider credsProvider) {

    if (!authState.isValid()) {
        return;
    }

    String hostname = host.getHostName();
    int port = host.getPort();
    if (port < 0) {
        Scheme scheme = connManager.getSchemeRegistry().getScheme(host);
        port = scheme.getDefaultPort();
    }

    AuthScheme authScheme = authState.getAuthScheme();
    AuthScope authScope = new AuthScope(
            hostname,
            port,
            authScheme.getRealm(),
            authScheme.getSchemeName());

    if (DEBUG) {
    	Logger.debug("Authentication scope: {}", authScope);
    }
    Credentials creds = authState.getCredentials();
    if (creds == null) {
        creds = credsProvider.getCredentials(authScope);
        if (DEBUG) {
            if (creds != null) {
            	Logger.debug("Found credentials");
            } else {
            	Logger.debug("Credentials not found");
            }
        }
    } else {
        if (authScheme.isComplete()) {
        	if (DEBUG) {
        		Logger.debug("Authentication failed");
        	}
            creds = null;
        }
    }
    authState.setAuthScope(authScope);
    authState.setCredentials(creds);
}
 

public static void main(String[] args) throws Exception {
	  OAuthConsumer consumer = new CommonsHttpOAuthConsumer(
				Constants.ConsumerKey,
				Constants.ConsumerSecret);
	  consumer.setTokenWithSecret(Constants.AccessToken, Constants.AccessSecret);
	  
	  
    HttpParams params = new BasicHttpParams();
    // HTTP 协议的版本,1.1/1.0/0.9
    HttpProtocolParams.setVersion(params, HttpVersion.HTTP_1_1);
    // 字符集
    HttpProtocolParams.setContentCharset(params, "UTF-8");
    // 伪装的浏览器类型
    // IE7 是 
    // Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 6.0)
    //
    // Firefox3.03
    // Mozilla/5.0 (Windows; U; Windows NT 5.2; zh-CN; rv:1.9.0.3) Gecko/2008092417 Firefox/3.0.3
    //
    HttpProtocolParams.setUserAgent(params, "HttpComponents/1.1");
    HttpProtocolParams.setUseExpectContinue(params, true);
    BasicHttpProcessor httpproc = new BasicHttpProcessor();
    httpproc.addInterceptor(new RequestContent());
    httpproc.addInterceptor(new RequestTargetHost());
    httpproc.addInterceptor(new RequestConnControl());
    httpproc.addInterceptor(new RequestUserAgent());
    httpproc.addInterceptor(new RequestExpectContinue());
    HttpRequestExecutor httpexecutor = new HttpRequestExecutor();
    HttpContext context = new BasicHttpContext(null);
    HttpHost host = new HttpHost("127.0.0.1", 1080);
    DefaultHttpClientConnection conn = new DefaultHttpClientConnection();
    ConnectionReuseStrategy connStrategy = new DefaultConnectionReuseStrategy();
    context.setAttribute(ExecutionContext.HTTP_CONNECTION, conn);
    context.setAttribute(ExecutionContext.HTTP_TARGET_HOST, host);
    try {
      String[] targets = { "https://www.twitter.com"};
      for (int i = 0; i < targets.length; i++) {
        if (!conn.isOpen()) {
          Socket socket = new Socket(host.getHostName(), host.getPort());
          conn.bind(socket, params);
        }
        
	  
        BasicHttpRequest request = new BasicHttpRequest("GET", targets[i]);
//        consumer.sign(request);
        
        System.out.println(">> Request URI: " + request.getRequestLine().getUri());
        context.setAttribute(ExecutionContext.HTTP_REQUEST, request);
        request.setParams(params);
        httpexecutor.preProcess(request, httpproc, context);
        HttpResponse response = httpexecutor.execute(request, conn, context);
        response.setParams(params);
        httpexecutor.postProcess(response, httpproc, context);
        // 返回码
        System.out.println("<< Response: " + response.getStatusLine());
        // 返回的文件头信息
//        Header[] hs = response.getAllHeaders();
//        for (Header h : hs) {
//          System.out.println(h.getName() + ":" + h.getValue());
//        }
        // 输出主体信息
//        System.out.println(EntityUtils.toString(response.getEntity()));
        
        HttpEntity entry = response.getEntity();
        StringBuffer sb = new StringBuffer();
  	  if(entry != null)
  	  {
  	    InputStreamReader is = new InputStreamReader(entry.getContent());
  	    BufferedReader br = new BufferedReader(is);
  	    String str = null;
  	    while((str = br.readLine()) != null)
  	    {
  	     sb.append(str.trim());
  	    }
  	    br.close();
  	  }
  	  System.out.println(sb.toString());
  	  
        System.out.println("==============");
        if (!connStrategy.keepAlive(response, context)) {
          conn.close();
        } else {
          System.out.println("Connection kept alive...");
        }
      }
    } finally {
      conn.close();
    }
  }
 

/**
 * 根据url下载网页内容
 *
 * @param url
 * @return
 */
public static String getHttpContent(String url) {

    CloseableHttpClient httpClient = null;
    HttpHost proxy = null;
    if (IPProxyRepository.size() > 0) {  // 如果ip代理地址库不为空，则设置代理
        proxy = getRandomProxy();
        httpClient = HttpClients.custom().setProxy(proxy).build();  // 创建httpclient对象
    } else {
        httpClient = HttpClients.custom().build();  // 创建httpclient对象
    }
    HttpGet request = new HttpGet(url); // 构建htttp get请求
    request.setHeader("user-agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.221 Safari/537.36 SE 2.X MetaSr 1.0");
    /*
    HttpHost proxy = null;
    CloseableHttpClient httpClient = HttpClients.custom().build();
    HttpGet request = new HttpGet(url); // 构建htttp get请求
    */
    /**
     * 设置超时时间
     * setConnectTimeout：设置连接超时时间，单位毫秒。
     * setConnectionRequestTimeout：设置从connect Manager获取Connection 超时时间，单位毫秒。这个属性是新加的属性，因为目前版本是可以共享连接池的。
     * setSocketTimeout：请求获取数据的超时时间，单位毫秒。 如果访问一个接口，多少时间内无法返回数据，就直接放弃此次调用。
     */
    RequestConfig requestConfig = RequestConfig.custom()
            .setConnectTimeout(5000).setConnectionRequestTimeout(1000)
            .setSocketTimeout(5000).build();
    request.setConfig(requestConfig);
    String host = null;
    Integer port = null;
    if(proxy != null) {
        host = proxy.getHostName();
        port = proxy.getPort();
    }
    try {
        long start = System.currentTimeMillis();    // 开始时间
        CloseableHttpResponse response = httpClient.execute(request);
        long end = System.currentTimeMillis();      // 结束时间
        logger.info("下载网页：{}，消耗时长：{} ms，代理信息：{}", url, end - start, host + ":" + port);
        return EntityUtils.toString(response.getEntity());
    } catch (Exception e) {
        logger.error("下载网页：{}出错，代理信息：{}，", url, host + ":" + port);
        // 如果该url为列表url，则将其添加到高优先级队列中
        if (url.contains("list.jd.com") || url.contains("list.suning.com")) {   // 这里为硬编码
            String domain = SpiderUtil.getTopDomain(url);   // jd.com
            retryUrl(url, domain + SpiderConstants.SPIDER_DOMAIN_HIGHER_SUFFIX);    // 添加url到jd.com.higher中
        }
        /**
         * 为什么要加入到高优先级队列中？
         * 如果该url为第一个种子url，但是解析却失败了，那么url仓库中会一直没有url，虽然爬虫程序还在执行，
         * 但是会一直提示没有url，这时就没有意义了，还需要尝试的另外一个原因是，下载网页失败很大可能是
         * 因为：
         *      1.此刻网络突然阻塞
         *      2.代理地址被限制，也就是被封了
         * 所以将其重新添加到高优先级队列中，再进行解析目前来说是比较不错的解决方案
         */
        e.printStackTrace();
    }

    return null;
}
 

static public AuthScope hostToAuthScope(HttpHost host) {
  return new AuthScope(host.getHostName(), host.getPort(), AuthScope.ANY_REALM, AuthScope.ANY_SCHEME);
}