Java Code Examples for org.apache.http.HttpEntity#getContent()

@Override
protected byte[] getResponseData(HttpEntity entity) throws IOException {
    if (entity != null) {
        InputStream instream = entity.getContent();
        long contentLength = entity.getContentLength() + current;
        FileOutputStream buffer = new FileOutputStream(getTargetFile(), append);
        if (instream != null) {
            try {
                byte[] tmp = new byte[BUFFER_SIZE];
                int l;
                while (current < contentLength && (l = instream.read(tmp)) != -1 && !Thread.currentThread().isInterrupted()) {
                    current += l;
                    buffer.write(tmp, 0, l);
                    sendProgressMessage((int) current, (int) contentLength);
                }
            } finally {
                instream.close();
                buffer.flush();
                buffer.close();
            }
        }
    }
    return null;
}
 

@Override
protected byte[] getResponseData(HttpEntity entity) throws IOException {
    if (entity != null) {
        InputStream instream = entity.getContent();
        long contentLength = entity.getContentLength();
        FileOutputStream buffer = new FileOutputStream(getTargetFile(), this.append);
        if (instream != null) {
            try {
                byte[] tmp = new byte[BUFFER_SIZE];
                int l, count = 0;
                // do not send messages if request has been cancelled
                while ((l = instream.read(tmp)) != -1 && !Thread.currentThread().isInterrupted()) {
                    count += l;
                    buffer.write(tmp, 0, l);
                    sendProgressMessage(count, (int) contentLength);
                }
            } finally {
                AsyncHttpClient.silentCloseInputStream(instream);
                buffer.flush();
                AsyncHttpClient.silentCloseOutputStream(buffer);
            }
        }
    }
    return null;
}
 

public static File download(URI uri, File dst, ProgressMonitor<Long> monitor) throws IOException {
   CloseableHttpResponse rsp = get(uri);
   try {
      if (rsp.getStatusLine().getStatusCode() != 200) {
         throw new IOException("http request failed: " + rsp.getStatusLine().getStatusCode());
      }

      HttpEntity entity = rsp.getEntity();
      long length = entity.getContentLength();

      try (InputStream is = entity.getContent();
           OutputStream os = new BufferedOutputStream(new FileOutputStream(dst))) {
         copy(is, os, length, monitor);
      }
      EntityUtils.consume(entity);
   } finally {
      rsp.close();
   }
   return dst;
}
 

public static HttpResult httpPost(String url, Map<String, String> headers, String content) {
    try {
        HttpClientBuilder builder = HttpClients.custom();
        builder.setConnectionTimeToLive(500, TimeUnit.MILLISECONDS);

        CloseableHttpClient httpClient = builder.build();
        HttpPost httpost = new HttpPost(url);

        if(headers != null){
            for (Map.Entry<String, String> entry : headers.entrySet()) {
                httpost.setHeader(entry.getKey(), entry.getValue());
            }
        }

        httpost.setEntity(new StringEntity(content, ContentType.create("application/json", DEFAULT_CONTENT_TYPE)));
        HttpResponse response = httpClient.execute(httpost);
        HttpEntity entity = response.getEntity();

        Reader reader = new InputStreamReader(entity.getContent(), DEFAULT_CONTENT_TYPE);
        CharArrayWriter sw = new CharArrayWriter();
        copy(reader, sw);
        return new HttpResult(response.getStatusLine().getStatusCode(), sw.toString(), Collections.emptyMap());
    } catch (Exception e) {
        return new HttpResult(500, e.toString(), Collections.emptyMap());
    }
}
 

public long[] translateKeys(Internal.TranslateKeysRequest request) throws IOException {
    String path = "/internal/translate/keys";
    ByteArrayEntity body = new ByteArrayEntity(request.toByteArray());
    CloseableHttpResponse response = clientExecute("POST", path, body, protobufHeaders, "Error while posting translateKey",
            ReturnClientResponse.RAW_RESPONSE, false);
    HttpEntity entity = response.getEntity();
    if (entity != null) {
        InputStream src = entity.getContent();
        Internal.TranslateKeysResponse translateKeysResponse = Internal.TranslateKeysResponse.parseFrom(src);
        List<Long> values = translateKeysResponse.getIDsList();
        long[] result = new long[values.size()];
        int i = 0;
        for (Long v : values) {
            result[i++] = v.longValue();
        }

        return result;

    }
    throw new PilosaException("Server returned empty response");
}
 

/**
 * This method is used to receive a stream of response body.
 * @param res Response object
 * @return Stream
 * @throws IOException Exception thrown
 */
protected final InputStream getResponseBodyInputStream(final HttpResponse res) throws IOException {
    // GZip 圧縮されていたら解凍する。
    /** thaw if it is GZip compression. */
    Header[] contentEncodingHeaders = res.getHeaders("Content-Encoding");
    if (contentEncodingHeaders.length > 0 && "gzip".equalsIgnoreCase(contentEncodingHeaders[0].getValue())) {
        return new GZIPInputStream(res.getEntity().getContent());
    } else {
        HttpEntity he = res.getEntity();
        if (he != null) {
            return he.getContent();
        } else {
            return null;
        }
    }
}
 

private void writeToDisk(HttpEntity entity, String file) throws IllegalStateException, IOException
/**
 * writes a HTTP entity to the specified filename and location on disk
 */
{
    //int i = 0;
    String FilePath = "/sdcard/" + file;
    InputStream in = entity.getContent();
    byte buff[] = new byte[1024];
    FileOutputStream out =
            new FileOutputStream(FilePath);
   do {
        int numread = in.read(buff);
        if (numread <= 0)
            break;
        out.write(buff, 0, numread);
        //i++;
    } while (true);
    out.flush();
    out.close();
}
 

@Override
public InputStream handleResponse(final HttpResponse response) throws IOException {
  final StatusLine statusLine = response.getStatusLine();
  final HttpEntity entity = response.getEntity();
  if (statusLine.getStatusCode() >= 300) {
    EntityUtils.consume(entity);
    throw new HttpResponseException(statusLine.getStatusCode(), statusLine.getReasonPhrase());
  }
  return entity == null ? null : entity.getContent();
}
 

protected String getASCIIContentFromEntity(HttpEntity entity) throws IllegalStateException, IOException {
	InputStream in = entity.getContent();
	StringBuffer out = new StringBuffer();
	int n = 1;
	while (n > 0) {
		byte[] b = new byte[4096];
		n = in.read(b);
		if (n > 0)
			out.append(new String(b, 0, n));
	}
	return out.toString();
}
 

public static String getAsString(URI uri)
        throws IOException {
    try (CloseableHttpResponse response = CLIENT.execute(new HttpGet(uri))) {
        HttpEntity entity = response.getEntity();
        String encoding = Optional.ofNullable(entity.getContentEncoding())
                .map(Header::getValue).orElse(StandardCharsets.UTF_8.name());
        Charset charset = Charset.forName(encoding);
        try (InputStream is = entity.getContent();
             Reader reader = new InputStreamReader(is, charset)) {
            return CharStreams.toString(reader);
        }
    }
}
 

private String fetchUserEmail(Integer githubId, String accessToken) {
	HttpGet emailsGet = new HttpGet(PROFILE_URL + "/emails");
	emailsGet.setHeader(HttpHeaders.AUTHORIZATION, "token " + accessToken);
	emailsGet.setHeader(HttpHeaders.ACCEPT, "application/json");
	String defaultEmail = githubId + "@github.com";
	try (CloseableHttpResponse resp = httpclient.execute(emailsGet)) {
		HttpEntity respEntity = resp.getEntity();
		if (respEntity != null) {
			try (InputStream is = respEntity.getContent()) {
				MappingIterator<Map<String, Object>> emails = jreader.readValues(is);
				if (emails != null) {
					String email = null;
					while (emails.hasNext()) {
						Map<String, Object> next = emails.next();
						email = (String) next.get("email");
						if (next.containsKey("primary") && (Boolean) next.get("primary")) {
							break;
						}
					}
					return email;
				}
			}
			EntityUtils.consumeQuietly(respEntity);
		}
	} catch (IOException e) {
		logger.warn("Failed to fetch user email from GitHub, using default: " + defaultEmail);
	}
	return defaultEmail;
}
 

/** Reads the contents of HttpEntity into a byte[]. */
private byte[] entityToBytes(HttpEntity entity) throws IOException, ServerError {
    PoolingByteArrayOutputStream bytes =
            new PoolingByteArrayOutputStream(mPool, (int) entity.getContentLength());
    byte[] buffer = null;
    try {
        InputStream in = entity.getContent();
        if (in == null) {
            throw new ServerError();
        }
        buffer = mPool.getBuf(1024);
        int count;
        while ((count = in.read(buffer)) != -1) {
            bytes.write(buffer, 0, count);
        }
        return bytes.toByteArray();
    } finally {
        try {
            // Close the InputStream and release the resources by "consuming the content".
            entity.consumeContent();
        } catch (IOException e) {
            // This can happen if there was an exception above that left the entity in
            // an invalid state.
            VolleyLog.v("Error occured when calling consumingContent");
        }
        mPool.returnBuf(buffer);
        bytes.close();
    }
}
 

/**
 * Overridden to provide a spot to modify the outbound response before its stream is closed.
 */
@Override
protected void writeOutboundResponse(HttpUriRequest outboundRequest, HttpServletRequest inboundRequest, HttpServletResponse outboundResponse, HttpResponse inboundResponse) throws IOException {
  // Copy the client respond header to the server respond.
  outboundResponse.setStatus(inboundResponse.getStatusLine().getStatusCode());
  Header[] headers = inboundResponse.getAllHeaders();
  Set<String> excludeHeaders = getOutboundResponseExcludeHeaders();
  boolean hasExcludeHeaders = false;
  if ((excludeHeaders != null) && !(excludeHeaders.isEmpty())) {
    hasExcludeHeaders = true;
  }
  for ( Header header : headers ) {
    String name = header.getName();
    if (hasExcludeHeaders && excludeHeaders.contains(name.toUpperCase(Locale.ROOT))) {
      continue;
    }
    String value = header.getValue();
    outboundResponse.addHeader(name, value);
  }

  HttpEntity entity = inboundResponse.getEntity();
  if( entity != null ) {
    outboundResponse.setContentType( getInboundResponseContentType( entity ) );
    InputStream stream = entity.getContent();
    try {
      NiFiResponseUtil.modifyOutboundResponse(inboundRequest, outboundResponse, inboundResponse);
      writeResponse( inboundRequest, outboundResponse, stream );
    } finally {
      closeInboundResponse( inboundResponse, stream );
    }
  }
}
 

/**
 * Returns byte array of response HttpEntity contents
 *
 * @param entity can be null
 * @return response entity body or null
 * @throws java.io.IOException if reading entity or creating byte array failed
 */
byte[] getResponseData(HttpEntity entity) throws IOException {
    byte[] responseBody = null;
    if (entity != null) {
        InputStream instream = entity.getContent();
        if (instream != null) {
            long contentLength = entity.getContentLength();
            if (contentLength > Integer.MAX_VALUE) {
                throw new IllegalArgumentException("HTTP entity too large to be buffered in memory");
            }
            int buffersize = (contentLength <= 0) ? BUFFER_SIZE : (int) contentLength;
            try {
                ByteArrayBuffer buffer = new ByteArrayBuffer(buffersize);
                try {
                    byte[] tmp = new byte[BUFFER_SIZE];
                    int l, count = 0;
                    // do not send messages if request has been cancelled
                    while ((l = instream.read(tmp)) != -1 && !Thread.currentThread().isInterrupted()) {
                        count += l;
                        buffer.append(tmp, 0, l);
                        sendProgressMessage(count, (int) (contentLength <= 0 ? 1 : contentLength));
                    }
                } finally {
                    AsyncHttpClient.silentCloseInputStream(instream);
                    AsyncHttpClient.endEntityViaReflection(entity);
                }
                responseBody = buffer.toByteArray();
            } catch (OutOfMemoryError e) {
                System.gc();
                throw new IOException("File too large to fit into available memory");
            }
        }
    }
    return responseBody;
}
 

/**
 * Returns input stream from defined url. Connection uses cache and eclipse proxy, if defined
 * @param url
 * @return url input stream
 * @throws IOException 
 */
public static InputStream getHttpStream(String url) throws IOException{
	URI target = URI.create(url);
	CloseableHttpClient client = getHttpClient(target);
	HttpGet get = new HttpGet(target);
	HttpResponse response = client.execute(get);
	HttpEntity entity = response.getEntity();
	StatusLine line = response.getStatusLine();
	if(line.getStatusCode() != 200){
		throw new IOException("HTTP response status is "+line.getStatusCode());
	}
	return entity.getContent();
}
 

@Override
public void run() {
    // check if query execution took longer than timeout
    if (this.timeOut < duration) {
        httpWorker.addResults(new QueryExecutionStats(queryId, COMMON.QUERY_SOCKET_TIMEOUT, duration));
        return;
    }

    // check if there was a problem with http response
    int responseCode = response.getStatusLine().getStatusCode();
    if (responseCode != 200) {
        httpWorker.addResults(new QueryExecutionStats(queryId, COMMON.QUERY_HTTP_FAILURE, duration));
        return;
    }

    // Check if the result of this query is already saved, if yes then use the saved result size instead of
    // processing the http response again
    HttpEntity httpResponse = response.getEntity();
    ConcurrentMap<QueryResultHashKey, Long> processedResults = httpWorker.getProcessedResults();
    QueryResultHashKey resultCacheKey = new QueryResultHashKey(queryId, httpResponse.getContentLength());
    if(processedResults.containsKey(resultCacheKey))
    {
        System.out.println("found : " + resultCacheKey);
        Long preCalculatedResultSize = processedResults.get(resultCacheKey);
        httpWorker.addResults(new QueryExecutionStats(queryId, COMMON.QUERY_SUCCESS, duration, preCalculatedResultSize));
        return;
    }

    // Result size is not saved before. Process the http response.
    Header[] contentTypeHeaders = response.getHeaders("Content-Type");
    String contentType = HttpWorker.getContentTypeVal(contentTypeHeaders[0]);

    try (InputStream inputStream = httpResponse.getContent();
         BufferedReader br = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) {
        StringBuilder result = new StringBuilder();
        String line;
        while ((line = br.readLine()) != null) {
            result.append(line);
        }
        System.out.println("[DEBUG]: byte size: " + result.length());

        long resultSize;
        if (HttpWorker.QUERY_RESULT_TYPE_JSON.equals(contentType)) {
            resultSize = HttpWorker.getJsonResultSize(result.toString());
        } else if (HttpWorker.QUERY_RESULT_TYPE_XML.equals(contentType)) {
            resultSize = HttpWorker.getXmlResultSize(result.toString());
        } else {
            resultSize = StringUtils.countMatches(result.toString(), "\n");
        }

        // Save the result size to be re-used
        processedResults.put(resultCacheKey, resultSize);
        System.out.println("added : " + resultCacheKey);

        response.close();
        client.close();

        httpWorker.addResults(new QueryExecutionStats(queryId, COMMON.QUERY_SUCCESS, duration, resultSize));

    } catch (IOException | ParseException | ParserConfigurationException | SAXException e) {
        System.out.println("Query results could not be parsed: " + e);
        httpWorker.addResults(new QueryExecutionStats(queryId, COMMON.QUERY_UNKNOWN_EXCEPTION, duration));
    }
}
 

/**
 * Create http POST request to an instagram api endpoint. 
 * 
 * @param requestUri Api url
 * @param params Request parameters
 * 	 
 * @return Api response in json format.
 * 
 * @throws Exception If error occured.
 */
public String post(String requestUrl, List<NameValuePair> params) throws Exception {
	InputStream stream 	= null;
	String response		= "";
	
	try {
		if (!mAccessToken.equals("")) {
			if (params == null) {
				params = new ArrayList<NameValuePair>(1);
				
				params.add(new BasicNameValuePair("access_token", mAccessToken));
			} else {
				params.add(new BasicNameValuePair("access_token", mAccessToken));
			}
		}
		
		Debug.i("POST " + requestUrl);
		
		HttpClient httpClient 	= new DefaultHttpClient();
		HttpPost httpPost 		= new HttpPost(requestUrl);
		
        httpPost.setEntity(new UrlEncodedFormEntity(params));
        
        HttpResponse httpResponse 	= httpClient.execute(httpPost);			
		HttpEntity httpEntity 		= httpResponse.getEntity();
		
		if (httpEntity == null) {
			throw new Exception("Request returns empty result");
		}
		
		stream		= httpEntity.getContent();			
		response	= StringUtil.streamToString(stream);
		
		Debug.i("Response " + response);
		
		if (httpResponse.getStatusLine().getStatusCode() != 200) {
			throw new Exception(httpResponse.getStatusLine().getReasonPhrase());
		}			
	} catch (Exception e) {
		throw e;
	}
	
	return response;
}
 

/** This function issues a Bing search API call and gets the JSONObject containing the relevant results
 * (including TotalCounts and SearchResults)
 * @param queryTerm (String) the term to query for.
 * @param count (int) URL parameter indicating how many results to return. Max value is 100.
 * @param offset (int) URL parameter indicating the offset for results.
 * @return a JSONObject containing the response.
 * @throws IOException
 */
private JsonNode fetchBingSearchAPIResponse(String queryTerm, Integer count, Integer offset) throws IOException {

  if (count <= 0) {
    LOGGER.error("Bing Search API was called with \"count\" URL parameter = 0. Please request at least one result.");
    return null;
  }

  URI uri = null;
  try {
    // Bing URL pattern. Note that we use composite queries to allow retrieval of the total results count.
    // Transaction cost is [count] bings, where [count] is the value of the URL parameter "count".
    // In other words, we can make 5M calls with [count]=1 per month.


    // Example: https://api.cognitive.microsoft.com/bing/v5.0/search?q=porsche&responseFilter=webpages
    uri = new URIBuilder()
        .setScheme("https")
        .setHost(BING_API_HOST)
        .setPath(BING_API_PATH)
        // Wrap the query term (%s) with double quotes (%%22) for exact search
        .setParameter("q", String.format("%s", queryTerm))
        // Restrict response to Web Pages only
        .setParameter("responseFilter", "webpages")
        // "count" parameter.
        .setParameter("count", count.toString())
        // "offset" parameter.
        .setParameter("offset", offset.toString())
        .build();

  } catch (URISyntaxException e) {
    LOGGER.error("An error occurred when trying to build the Bing Search API URI", e);
  }

  JsonNode results;
  HttpGet httpget = new HttpGet(uri);
  // Yay for un-encrypted account key!
  // TODO: actually is there a way to encrypt it?
  httpget.setHeader("Ocp-Apim-Subscription-Key", accountKey);

  CloseableHttpClient httpclient = HttpClients.custom().setConnectionManager(basicConnManager).build();

  try (CloseableHttpResponse response = httpclient.execute(httpget)) {
    Integer statusCode = response.getStatusLine().getStatusCode();

    // TODO: The Web Search API returns useful error messages, we could use them to have better insights on failures.
    // See: https://dev.cognitive.microsoft.com/docs/services/56b43eeccf5ff8098cef3807/operations/56b4447dcf5ff8098cef380d
    if (!statusCode.equals(HttpStatus.SC_OK)) {
      LOGGER.error("Bing Search API call returned an unexpected status code (%d) for URI: %s", statusCode, uri);
      return null;
    }

    HttpEntity entity = response.getEntity();
    ContentType contentType = ContentType.getOrDefault(entity);
    Charset charset = contentType.getCharset();
    if (charset == null) {
      charset = StandardCharsets.UTF_8;
    }

    try (final BufferedReader in = new BufferedReader(new InputStreamReader(entity.getContent(), charset))) {
      String inputLine;
      final StringBuilder stringResponse = new StringBuilder();
      while ((inputLine = in.readLine()) != null) {
        stringResponse.append(inputLine);
      }
      JsonNode rootNode = mapper.readValue(stringResponse.toString(), JsonNode.class);
      results = rootNode.path("webPages");
    }
  }
  return results;
}
 

private void handleQuery(String queryHandlerPath, HttpServletRequest request, HttpServletResponse resp)
    throws IOException, TransformerException, SAXException {
    ModifiableSolrParams solrParameter = getSolrQueryParameter(request);
    HttpGet solrHttpMethod = MCRSolrProxyServlet.getSolrHttpMethod(queryHandlerPath, solrParameter,
        Optional.ofNullable(request.getParameter(QUERY_CORE_PARAMETER)).orElse(MCRSolrConstants.MAIN_CORE_TYPE));
    try {
        LOGGER.info("Sending Request: {}", solrHttpMethod.getURI());
        HttpResponse response = httpClient.execute(solrHttpMethod);
        int statusCode = response.getStatusLine().getStatusCode();

        // set status code
        resp.setStatus(statusCode);

        boolean isXML = response.getFirstHeader(HTTP.CONTENT_TYPE).getValue().contains("/xml");
        boolean justCopyInput = !isXML;

        // set all headers
        for (Header header : response.getAllHeaders()) {
            LOGGER.debug("SOLR response header: {} - {}", header.getName(), header.getValue());
            String headerName = header.getName();
            if (NEW_HTTP_RESPONSE_HEADER.containsKey(headerName)) {
                String headerValue = NEW_HTTP_RESPONSE_HEADER.get(headerName);
                if (headerValue != null && headerValue.length() > 0) {
                    resp.setHeader(headerName, headerValue);
                }
            } else {
                resp.setHeader(header.getName(), header.getValue());
            }
        }

        HttpEntity solrResponseEntity = response.getEntity();
        if (solrResponseEntity != null) {
            try (InputStream solrResponseStream = solrResponseEntity.getContent()) {
                if (justCopyInput) {
                    // copy solr response to servlet outputstream
                    OutputStream servletOutput = resp.getOutputStream();
                    IOUtils.copy(solrResponseStream, servletOutput);
                } else {
                    MCRStreamContent solrResponse = new MCRStreamContent(solrResponseStream,
                        solrHttpMethod.getURI().toString(),
                        "response");
                    MCRLayoutService.instance().doLayout(request, resp, solrResponse);
                }
            }
        }
    } catch (IOException ex) {
        solrHttpMethod.abort();
        throw ex;
    }
    solrHttpMethod.releaseConnection();
}
 

private String getResponseFromEntity(HttpEntity entity) throws Exception {
		StringBuffer buffer = new StringBuffer();
		if (entity != null) {
			BufferedReader reader = new BufferedReader(new InputStreamReader(
					entity.getContent(), "utf-8"), 8192);
			String line = null;
			while ((line = reader.readLine()) != null) {
				buffer.append(line);
			}
			reader.close();
		}
		return buffer.toString();
	}