Java Code Examples for org.apache.http.util.EntityUtils#consumeQuietly()

@Test
public void testWaitForQuiescenceAfterRequestCompleted() throws IOException {
    String url = "/quiescencecompleted";

    stubFor(get(urlEqualTo(url)).willReturn(ok()));

    try (CloseableHttpClient client = NewProxyServerTestUtil.getNewHttpClient(proxy.getPort())) {
        HttpResponse response = client.execute(new HttpGet("http://127.0.0.1:" + mockServerPort + "/quiescencecompleted"));
        EntityUtils.consumeQuietly(response.getEntity());

        assertEquals("Expected successful response from server", 200, response.getStatusLine().getStatusCode());
    }

    // wait for 2s of quiescence, now that the call has already completed
    long start = System.nanoTime();
    boolean waitSuccessful = proxy.waitForQuiescence(2, 5, TimeUnit.SECONDS);
    long finish = System.nanoTime();

    assertTrue("Expected to successfully wait for quiescence", waitSuccessful);

    assertTrue("Expected to wait for quiescence for approximately 2s. Actual wait time was: " + TimeUnit.MILLISECONDS.convert(finish - start, TimeUnit.NANOSECONDS) + "ms",
            TimeUnit.MILLISECONDS.convert(finish - start, TimeUnit.NANOSECONDS) >= 1500 && TimeUnit.MILLISECONDS.convert(finish - start, TimeUnit.NANOSECONDS) <= 2500);

    verify(1, getRequestedFor(urlEqualTo(url)));
}
 

/**
 * get a redirect for an url: this method shall be called if it is expected that a url
 * is redirected to another url. This method then discovers the redirect.
 * @param urlstring
 * @param useAuthentication
 * @return the redirect url for the given urlstring
 * @throws IOException if the url is not redirected
 */
public static String getRedirect(String urlstring) throws IOException {
    HttpGet get = new HttpGet(urlstring);
    get.setConfig(RequestConfig.custom().setRedirectsEnabled(false).build());
    get.setHeader("User-Agent", ClientIdentification.getAgent(ClientIdentification.yacyInternetCrawlerAgentName).userAgent);
    CloseableHttpClient httpClient = getClosableHttpClient();
    HttpResponse httpResponse = httpClient.execute(get);
    HttpEntity httpEntity = httpResponse.getEntity();
    if (httpEntity != null) {
        if (httpResponse.getStatusLine().getStatusCode() == 301) {
            for (Header header: httpResponse.getAllHeaders()) {
                if (header.getName().equalsIgnoreCase("location")) {
                    EntityUtils.consumeQuietly(httpEntity);
                    return header.getValue();
                }
            }
            EntityUtils.consumeQuietly(httpEntity);
            throw new IOException("redirect for  " + urlstring+ ": no location attribute found");
        } else {
            EntityUtils.consumeQuietly(httpEntity);
            throw new IOException("no redirect for  " + urlstring+ " fail: " + httpResponse.getStatusLine().getStatusCode() + ": " + httpResponse.getStatusLine().getReasonPhrase());
        }
    } else {
        throw new IOException("client connection to " + urlstring + " fail: no connection");
    }
}
 

public void flush() throws IOException {
  if (byteBuffer.position() == 0) {
    return;
  }
  byteBuffer.flip();
  HttpPost httpPost = new HttpPost(this.dbUri);
  httpPost.setEntity(
      new ByteArrayEntity(byteBuffer.array(), 0, byteBuffer.limit(), ContentType.DEFAULT_TEXT));
  try {
    CloseableHttpResponse response = httpClient.execute(httpPost);
    EntityUtils.consumeQuietly(response.getEntity());
    int statusCode = response.getStatusLine().getStatusCode();
    if (statusCode / 100 != 2) {
      throw new IOException(
          "InfluxDB write failed: " + statusCode + " " + response.getStatusLine()
              .getReasonPhrase());
    }
  } finally {
    // Always clear the buffer. But this will lead to data loss in case of non 2xx response (i.e write operation failed)
    // received from the InfluxDB server. Ideally non 2xx server response should be rare but revisit this part
    // if data loss occurs frequently.
    byteBuffer.clear();
  }
}
 

private void httpClientDownload(HandlerContext ctx, Seed seed) {
    Payload payload = ctx.cetty().getPayload();

    CloseableHttpClient httpClient = ctx.cetty().getHttpClient();
    CloseableHttpResponse httpResponse = null;

    Page page;
    try {
        httpResponse = httpClient.execute(convertHttpUriRequest(seed, payload), convertHttpClientContext(seed, payload));
        page = handleResponse(seed, seed.getCharset() != null ? seed.getCharset() : payload.getCharset(), httpResponse);
        logger.info("download {} page success !", seed.getUrl());
        ctx.fireProcess(page);
    } catch (IOException e) {
        logger.warn("download {} page error !", seed.getUrl(), e);
    } finally {
        if (httpResponse != null) {
            EntityUtils.consumeQuietly(httpResponse.getEntity());
        }
    }

}
 

protected void close(HttpResponse response) {
    if (response == null)
        return;

    HttpEntity entity = response.getEntity();
    if (entity != null)
        EntityUtils.consumeQuietly(entity);
}
 

private void runTest(int concurrency, HttpHandler handler) throws IOException, InterruptedException {
    DefaultServer.setRootHandler(handler);
    DefaultServer.startSSLServer();
    try (CloseableHttpClient client = HttpClients.custom().disableConnectionState()
            .setSSLContext(DefaultServer.getClientSSLContext())
            .setDefaultSocketConfig(SocketConfig.custom().setSoTimeout(5000).build())
            .setMaxConnPerRoute(1000)
            .build()) {
        ExecutorService executorService = Executors.newFixedThreadPool(concurrency);
        AtomicBoolean failed = new AtomicBoolean();
        Runnable task = new Runnable() {
            @Override
            public void run() {
                if (failed.get()) {
                    return;
                }
                try (CloseableHttpResponse result = client.execute(new HttpGet(DefaultServer.getDefaultServerSSLAddress()))) {
                    Assert.assertEquals(StatusCodes.OK, result.getStatusLine().getStatusCode());
                    Header[] header = result.getHeaders("scheme");
                    Assert.assertEquals("https", header[0].getValue());
                    EntityUtils.consumeQuietly(result.getEntity());
                } catch (Throwable t) {
                    if (failed.compareAndSet(false, true)) {
                        t.printStackTrace();
                        executorService.shutdownNow();
                    }
                }
            }
        };
        for (int i = 0; i < concurrency * 300; i++) {
            executorService.submit(task);
        }
        executorService.shutdown();
        Assert.assertTrue(executorService.awaitTermination(70, TimeUnit.SECONDS));
        Assert.assertFalse(failed.get());
    } finally {
        DefaultServer.stopSSLServer();
    }
}
 

public void close() {
    HttpEntity httpEntity = this.httpResponse.getEntity();
    if (httpEntity != null) EntityUtils.consumeQuietly(httpEntity);
    try {
        this.inputStream.close();
    } catch (IOException e) {} finally {
        this.request.releaseConnection();
    }
}
 

@Override
public void completed(HttpResponse httpResponse) {
    try {
        Page page = handleResponse(seed, seed.getCharset() != null ? seed.getCharset() : payload.getCharset(), httpResponse);
        logger.info("download {} page success !", seed.getUrl());
        ctx.fireProcess(page);
    } catch (IOException e) {
        logger.warn("download {} page error !", seed.getUrl(), e);
    } finally {
        if (httpResponse != null) {
            EntityUtils.consumeQuietly(httpResponse.getEntity());
        }
    }
}
 

@Override
public byte[] performQuery(RequestMethod requestMethod, String path, Map<String, Object> data)
    throws
    DatabricksRestException {

  HttpRequestBase method = null;
  try {
    method = makeHttpMethod(requestMethod, path, data);

    // set authorization header if token base
    if (isTokenAuth) {
      method.addHeader("Authorization", String.format("Bearer %s", authToken));
    }

    HttpResponse httpResponse = client.execute(method);

    byte[] response = extractContent(httpResponse);

    EntityUtils.consumeQuietly(httpResponse.getEntity());

    return response;

  } catch (DatabricksRestException dre) {
    throw dre;
  } catch (Exception e) {
    throw new DatabricksRestException(e);
  } finally {
    if (method != null) {
      method.releaseConnection();
    }
  }
}
 

@Override
public byte[] performQuery(RequestMethod requestMethod, String path, Map<String, Object> data)
    throws
    DatabricksRestException {

  CloseableHttpResponse httpResponse = null;
  try {
    HttpRequestBase method = makeHttpMethod(requestMethod, path, data);
    httpResponse = ((CloseableHttpClient) client).execute(method);

    byte[] response = extractContent(httpResponse);

    EntityUtils.consumeQuietly(httpResponse.getEntity());

    return response;
  } catch (DatabricksRestException dre) {
    throw dre;
  } catch (Exception e) {
    throw new DatabricksRestException(e);
  } finally {
    try {
      if (httpResponse != null) {
        httpResponse.close();
      }
    } catch (IOException ioe) {
      logger.debug("ignore close error", ioe);
    }
  }
}
 

@Override
public Page download(Request request, Task task) {
    if (task == null || task.getSite() == null) {
        throw new NullPointerException("task or site can not be null");
    }
    CloseableHttpResponse httpResponse = null;
    CloseableHttpClient httpClient = getHttpClient(task.getSite());
    Proxy proxy = proxyProvider != null ? proxyProvider.getProxy(task) : null;
    HttpClientRequestContext requestContext = httpUriRequestConverter.convert(request, task.getSite(), proxy);
    Page page = Page.fail();
    try {
        httpResponse = httpClient.execute(requestContext.getHttpUriRequest(), requestContext.getHttpClientContext());
        page = handleResponse(request, request.getCharset() != null ? request.getCharset() : task.getSite().getCharset(), httpResponse, task);
        onSuccess(request);
        logger.info("downloading page success {}", request.getUrl());
        return page;
    } catch (IOException e) {
        logger.warn("download page {} error", request.getUrl(), e);
        onError(request);
        return page;
    } finally {
        if (httpResponse != null) {
            //ensure the connection is released back to pool
            EntityUtils.consumeQuietly(httpResponse.getEntity());
        }
        if (proxyProvider != null && proxy != null) {
            proxyProvider.returnProxy(proxy, page, task);
        }
    }
}
 

private static ProactiveCreds createCreds(CloseableHttpResponse response) throws IOException {
   HttpEntity entity = response.getEntity();
   String responseBody = EntityUtils.toString(entity, StandardCharsets.UTF_8);
   EntityUtils.consumeQuietly(entity);
   TokenInfo tokenInfo = JSON.fromJson(responseBody, TokenInfo.class);
   Date expiresIn = new Date(System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(tokenInfo.getExpires_in()));
   return new ProactiveCreds(tokenInfo.getAccess_token(), expiresIn, tokenInfo.getRefresh_token());
}
 

private void doHttpClientRequest(HttpClient httpClient, HttpUriRequest request) throws Exception {
  HttpResponse response = null;
  try {
    response = httpClient.execute(request);
    final int httpStatus = response.getStatusLine().getStatusCode();
    Assert.assertEquals(HttpURLConnection.HTTP_OK, httpStatus);
  } finally {
    if (response != null) EntityUtils.consumeQuietly(response.getEntity());
  }
}
 

@Override
public Page download(Request request, Task task) {
    if (task == null || task.getSite() == null) {
        throw new NullPointerException("task or site can not be null");
    }
    CloseableHttpResponse httpResponse = null;
    CloseableHttpClient httpClient = getHttpClient(task.getSite());
    Proxy proxy = proxyProvider != null ? proxyProvider.getProxy(task) : null;
    HttpClientRequestContext requestContext = httpUriRequestConverter.convert(request, task.getSite(), proxy);
    Page page = Page.fail();
    try {
        httpResponse = httpClient.execute(requestContext.getHttpUriRequest(), requestContext.getHttpClientContext());
        page = handleResponse(request, request.getCharset() != null ? request.getCharset() : task.getSite().getCharset(), httpResponse, task);
        onSuccess(request);
        logger.debug("downloading page success {}", request.getUrl());
        return page;
    } catch (IOException e) {
        logger.warn("download page {} error", request.getUrl(), e);
        onError(request);
        return page;
    } finally {
        if (httpResponse != null) {
            //ensure the connection is released back to pool
            EntityUtils.consumeQuietly(httpResponse.getEntity());
        }
        if (proxyProvider != null && proxy != null) {
            proxyProvider.returnProxy(proxy, page, task);
        }
    }
}
 

void flush() throws IOException {
  if (currentBatchSize < 1) {
    return;
  }

  // flush
  writer.write(']');
  writer.flush();
  HttpPost httpPost = new HttpPost(dbUri);
  httpPost.setEntity(new ByteArrayEntity(buffer.toByteArray(), ContentType.APPLICATION_JSON));
  final StatusLine status;
  CloseableHttpResponse response = null;
  try {
    response = httpClient.execute(httpPost);
    status = response.getStatusLine();
    // CLOVER:OFF
    // Just tracing.
    if (LOG.isTraceEnabled()) {
      LOG.trace("Response from OpenTSDB [{}] ",
          EntityUtils.toString(response.getEntity()));
    }
    // CLOVER:ON
  } finally {
    if (response != null) {
      EntityUtils.consumeQuietly(response.getEntity());
    }
    // reset our buffer and batch size
    currentBatchSize = 0;
    buffer.reset();
    writer.write('[');
  }
  if (status.getStatusCode() / 100 != 2) {
    throw new IllegalStateException(String.format(
        "Failed to write metrics to OpenTSDB '%d' - '%s'",
        status.getStatusCode(),
        status.getReasonPhrase()));
  }
}
 

@Test
public void testWaitForQuiescenceTimeoutLessThanQuietPeriodUnuccessful() throws IOException, InterruptedException {
    String url = "/quiescencesmalltimeoutunsuccessful";

    stubFor(get(urlEqualTo(url)).willReturn(ok()));

    try (CloseableHttpClient client = NewProxyServerTestUtil.getNewHttpClient(proxy.getPort())) {
        HttpResponse response = client.execute(new HttpGet("http://127.0.0.1:" + mockServerPort + "/quiescencesmalltimeoutunsuccessful"));
        EntityUtils.consumeQuietly(response.getEntity());

        assertEquals("Expected successful response from server", 200, response.getStatusLine().getStatusCode());
    }

    Thread.sleep(1000);

    // wait for 3s of quiescence within 1s, which should not be possible since the last request just finished. waitForQuiescence should
    // be able to detect that and return immediately.
    long start = System.nanoTime();
    boolean waitSuccessful = proxy.waitForQuiescence(3, 1, TimeUnit.SECONDS);
    long finish = System.nanoTime();

    assertFalse("Expected to unsuccessfully wait for quiescence", waitSuccessful);

    assertTrue("Expected wait for quiescence to return immediately. Actual wait time was: " + TimeUnit.MILLISECONDS.convert(finish - start, TimeUnit.NANOSECONDS) + "ms",
            TimeUnit.MILLISECONDS.convert(finish - start, TimeUnit.NANOSECONDS) >= 0 && TimeUnit.MILLISECONDS.convert(finish - start, TimeUnit.NANOSECONDS) <= 10);

    verify(1, getRequestedFor(urlEqualTo(url)));
}
 

@Test
public void testWaitForQuiescenceQuietPeriodAlreadySatisfied() throws IOException, InterruptedException {
    String url = "/quiescencesatisfied";

    stubFor(get(urlEqualTo(url)).willReturn(ok()));

    try (CloseableHttpClient client = NewProxyServerTestUtil.getNewHttpClient(proxy.getPort())) {
        HttpResponse response = client.execute(new HttpGet("http://127.0.0.1:" + mockServerPort + "/quiescencesatisfied"));
        EntityUtils.consumeQuietly(response.getEntity());

        assertEquals("Expected successful response from server", 200, response.getStatusLine().getStatusCode());
    }

    // wait for 2s, then wait for 1s of quiescence, which should already be satisfied
    Thread.sleep(2000);

    long start = System.nanoTime();
    boolean waitSuccessful = proxy.waitForQuiescence(1, 5, TimeUnit.SECONDS);
    long finish = System.nanoTime();

    assertTrue("Expected to successfully wait for quiescence", waitSuccessful);

    assertTrue("Expected wait for quiescence to return immediately. Actual wait time was: " + TimeUnit.MILLISECONDS.convert(finish - start, TimeUnit.NANOSECONDS) + "ms",
            TimeUnit.MILLISECONDS.convert(finish - start, TimeUnit.NANOSECONDS) <= 1);

    verify(1, getRequestedFor(urlEqualTo(url)));
}
 

private void consumeQuietly(HttpEntity entity) {
   if(entity != null) {
      EntityUtils.consumeQuietly(entity);
   }
}
 

@Override
public Page download(Request request, Task task) {
    if (task == null || task.getSite() == null) {
        throw new NullPointerException("task or site can not be null");
    }
    CloseableHttpResponse httpResponse = null;
    CloseableHttpClient httpClient = getHttpClient(task.getSite());
    Proxy proxy = proxyProvider != null ? proxyProvider.getProxy(task) : null;
    HttpClientRequestContext requestContext = httpUriRequestConverter.convert(request, task.getSite(), proxy);
    Page page = Page.fail();
    try {
        httpResponse = httpClient.execute(requestContext.getHttpUriRequest(), requestContext.getHttpClientContext());
        page = handleResponse(request, request.getCharset() != null ? request.getCharset() : task.getSite().getCharset(), httpResponse, task);
        onSuccess(request);
        logger.debug("downloading page success {}", request.getUrl());
    } catch (IOException e) {
        if (e instanceof ConnectionClosedException) {
            logger.error("Premature end of chunk coded message body: {}", request.getUrl());
        } else if (e instanceof SSLHandshakeException) {
            logger.error("Remote host closed connection during handshake: {}", request.getUrl());
        } else if (e instanceof SSLException) {
            logger.error("SSL peer shut down incorrectly:[HttpClient]  {}", request.getUrl());
        } else if (e instanceof SocketTimeoutException) {
            logger.error("download page time out:{}", request.getUrl());
        } else if (e instanceof NoHttpResponseException) {
            logger.error("failed to respond:{}", request.getUrl());
        } else if (e instanceof HttpHostConnectException) {
            logger.error("Connect to proxy timed out:{}", request.getUrl());
        } else if (e instanceof TruncatedChunkException) {
            logger.error("TruncatedChunkException:{}, msg:{}", request.getUrl(), e.getMessage());
        } else {
            logger.error("download page error:{} ", request.getUrl(), e);
        }
        onError(request);
    } finally {
        if (httpResponse != null) {
            //ensure the connection is released back to pool
            EntityUtils.consumeQuietly(httpResponse.getEntity());
        }
        if (proxyProvider != null && proxy != null) {
            proxyProvider.returnProxy(proxy, page, task);
        }
    }
    return page;
}
 

@Test
public void testWaitForQuiescenceTimeoutLessThanQuietPeriodSuccessful() throws IOException, InterruptedException {
    String url = "/quiescencesmalltimeoutsuccess";

    stubFor(get(urlEqualTo(url)).willReturn(ok()));

    try (CloseableHttpClient client = NewProxyServerTestUtil.getNewHttpClient(proxy.getPort())) {
        HttpResponse response = client.execute(new HttpGet("http://127.0.0.1:" + mockServerPort + "/quiescencesmalltimeoutsuccess"));
        EntityUtils.consumeQuietly(response.getEntity());

        assertEquals("Expected successful response from server", 200, response.getStatusLine().getStatusCode());
    }

    Thread.sleep(2500);

    // wait for 3s of quiescence, which should wait no more than 500ms

    long start = System.nanoTime();
    boolean waitSuccessful = proxy.waitForQuiescence(3, 1, TimeUnit.SECONDS);
    long finish = System.nanoTime();

    assertTrue("Expected to successfully wait for quiescence", waitSuccessful);

    assertTrue("Expected to wait for quiescence for approximately 500ms. Actual wait time was: " + TimeUnit.MILLISECONDS.convert(finish - start, TimeUnit.NANOSECONDS) + "ms",
            TimeUnit.MILLISECONDS.convert(finish - start, TimeUnit.NANOSECONDS) >= 300 && TimeUnit.MILLISECONDS.convert(finish - start, TimeUnit.NANOSECONDS) <= 700);

    verify(1, getRequestedFor(urlEqualTo(url)));
}