com.google.appengine.api.urlfetch.HTTPResponse Java Examples

@Test
public void testDescribeRequestAndResponseF() throws Exception {
  HTTPRequest request = new HTTPRequest(new URL("http://ping/pong"));
  request.setPayload("hello".getBytes());
  request.addHeader(new HTTPHeader("k1", "v1"));
  request.addHeader(new HTTPHeader("k2", "v2"));
  HTTPResponse response = mock(HTTPResponse.class);
  when(response.getHeadersUncombined()).thenReturn(ImmutableList.of(new HTTPHeader("k3", "v3")));
  when(response.getResponseCode()).thenReturn(500);
  when(response.getContent()).thenReturn("bla".getBytes());
  String expected = "Request: GET http://ping/pong\nk1: v1\nk2: v2\n\n"
      + "5 bytes of content\n\nResponse: 500 with 3 bytes of content\nk3: v3\nbla\n";
  String result =
      URLFetchUtils.describeRequestAndResponse(new HTTPRequestInfo(request), response);
  assertEquals(expected, result);
}
 

@Override
public RawGcsCreationToken beginObjectCreation(
    GcsFilename filename, GcsFileOptions options, long timeoutMillis) throws IOException {
  HTTPRequest req = makeRequest(filename, null, POST, timeoutMillis);
  req.setHeader(RESUMABLE_HEADER);
  addOptionsHeaders(req, options);
  HTTPResponse resp;
  try {
    resp = urlfetch.fetch(req);
  } catch (IOException e) {
    throw createIOException(new HTTPRequestInfo(req), e);
  }
  if (resp.getResponseCode() == 201) {
    String location = URLFetchUtils.getSingleHeader(resp, LOCATION);
    String queryString = new URL(location).getQuery();
    Preconditions.checkState(
        queryString != null, LOCATION + " header," + location + ", witout a query string");
    Map<String, String> params = Splitter.on('&').withKeyValueSeparator('=').split(queryString);
    Preconditions.checkState(params.containsKey(UPLOAD_ID),
        LOCATION + " header," + location + ", has a query string without " + UPLOAD_ID);
    return new GcsRestCreationToken(filename, params.get(UPLOAD_ID), 0);
  } else {
    throw HttpErrorHandler.error(new HTTPRequestInfo(req), resp);
  }
}
 

@Test
public void testAsyncOps() throws Exception {
    URLFetchService service = URLFetchServiceFactory.getURLFetchService();

    URL adminConsole = findAvailableUrl(URLS);
    Future<HTTPResponse> response = service.fetchAsync(adminConsole);
    printResponse(response.get(5, TimeUnit.SECONDS));

    response = service.fetchAsync(new HTTPRequest(adminConsole));
    printResponse(response.get(5, TimeUnit.SECONDS));

    URL jbossOrg = new URL("http://www.jboss.org");
    if (available(jbossOrg)) {
        response = service.fetchAsync(jbossOrg);
        printResponse(response.get(30, TimeUnit.SECONDS));
    }

    sync(5000L); // wait a bit for async to finish
}
 

/**
 * Given a HTTPResponce, process it, throwing an error if needed and return a Token for the next
 * request.
 */
GcsRestCreationToken handlePutResponse(final GcsRestCreationToken token,
    final boolean isFinalChunk,
    final int length,
    final HTTPRequestInfo reqInfo,
    HTTPResponse resp) throws Error, IOException {
  switch (resp.getResponseCode()) {
    case 200:
      if (!isFinalChunk) {
        throw new RuntimeException("Unexpected response code 200 on non-final chunk. Request: \n"
            + URLFetchUtils.describeRequestAndResponse(reqInfo, resp));
      } else {
        return null;
      }
    case 308:
      if (isFinalChunk) {
        throw new RuntimeException("Unexpected response code 308 on final chunk: "
            + URLFetchUtils.describeRequestAndResponse(reqInfo, resp));
      } else {
        return new GcsRestCreationToken(token.filename, token.uploadId, token.offset + length);
      }
    default:
      throw HttpErrorHandler.error(resp.getResponseCode(),
          URLFetchUtils.describeRequestAndResponse(reqInfo, resp));
  }
}
 

@Override
public T await() throws ApiException, IOException, InterruptedException {
  try {
    HTTPResponse result = call.get();
    metrics.endNetwork();
    return parseResponse(this, result);
  } catch (ExecutionException e) {
    if (e.getCause() instanceof IOException) {
      throw (IOException) e.getCause();
    } else {
      // According to
      // https://cloud.google.com/appengine/docs/standard/java/javadoc/com/google/appengine/api/urlfetch/URLFetchService
      // all exceptions should be subclass of IOException so this should not happen.
      throw new UnknownErrorException("Unexpected exception from " + e.getMessage());
    }
  }
}
 

public void doGet(HttpServletRequest req, HttpServletResponse res)
    throws IOException, ServletException {
  String url = req.getParameter("url");
  String deadlineSecs = req.getParameter("deadline");
  URLFetchService service = URLFetchServiceFactory.getURLFetchService();
  HTTPRequest fetchReq = new HTTPRequest(new URL(url));
  if (deadlineSecs != null) {
    fetchReq.getFetchOptions().setDeadline(Double.valueOf(deadlineSecs));
  }
  HTTPResponse fetchRes = service.fetch(fetchReq);
  for (HTTPHeader header : fetchRes.getHeaders()) {
    res.addHeader(header.getName(), header.getValue());
  }
  if (fetchRes.getResponseCode() == 200) {
    res.getOutputStream().write(fetchRes.getContent());
  } else {
    res.sendError(fetchRes.getResponseCode(), "Error while fetching");
  }
}
 

UrlFetchResponse(HTTPResponse fetchResponse) {
  this.fetchResponse = fetchResponse;
  for (HTTPHeader header : fetchResponse.getHeadersUncombined()) {
    String name = header.getName();
    String value = header.getValue();
    if (name != null && value != null) {
      headerNames.add(name);
      headerValues.add(value);
      if ("content-type".equalsIgnoreCase(name)) {
        contentType = value;
      } else if ("content-encoding".equalsIgnoreCase(name)) {
        contentEncoding = value;
      } else if ("content-length".equalsIgnoreCase(name)) {
        try {
          contentLength = Long.parseLong(value);
        } catch (NumberFormatException e) {
          // ignore
        }
      }
    }
  }
}
 

@Override
public LowLevelHttpResponse execute() throws IOException {
  // write content
  if (getStreamingContent() != null) {
    String contentType = getContentType();
    if (contentType != null) {
      addHeader("Content-Type", contentType);
    }
    String contentEncoding = getContentEncoding();
    if (contentEncoding != null) {
      addHeader("Content-Encoding", contentEncoding);
    }
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    getStreamingContent().writeTo(out);
    byte[] payload = out.toByteArray();
    if (payload.length != 0) {
      request.setPayload(payload);
    }
  }
  // connect
  URLFetchService service = URLFetchServiceFactory.getURLFetchService();
  HTTPResponse response = service.fetch(request);
  return new UrlFetchResponse(response);
}
 

/**
 * Same as {@link #put} but is runs asynchronously and returns a future. In the event of an error
 * the exception out of the future will be an ExecutionException with the cause set to the same
 * exception that would have been thrown by put.
 */
private Future<RawGcsCreationToken> putAsync(final GcsRestCreationToken token,
    ByteBuffer chunk, final boolean isFinalChunk, long timeoutMillis) {
  final int length = chunk.remaining();
  HTTPRequest request = createPutRequest(token, chunk, isFinalChunk, timeoutMillis, length);
  final HTTPRequestInfo info = new HTTPRequestInfo(request);
  return new FutureWrapper<HTTPResponse, RawGcsCreationToken>(urlfetch.fetchAsync(request)) {
    @Override
    protected Throwable convertException(Throwable e) {
      return OauthRawGcsService.convertException(info, e);
    }

    @Override
    protected GcsRestCreationToken wrap(HTTPResponse resp) throws Exception {
      return handlePutResponse(token, isFinalChunk, length, info, resp);
    }
  };
}
 

/** True if deleted, false if not found. */
@Override
public boolean deleteObject(GcsFilename filename, long timeoutMillis) throws IOException {
  HTTPRequest req = makeRequest(filename, null, DELETE, timeoutMillis);
  HTTPResponse resp;
  try {
    resp = urlfetch.fetch(req);
  } catch (IOException e) {
    throw createIOException(new HTTPRequestInfo(req), e);
  }
  switch (resp.getResponseCode()) {
    case 204:
      return true;
    case 404:
      return false;
    default:
      throw HttpErrorHandler.error(new HTTPRequestInfo(req), resp);
  }
}
 

public static String getResponseDELETE(String url, Map<String, String> params, Map<String, String> headers) {
	int retry = 0;
	while (retry < 3) {
		long start = System.currentTimeMillis();
		try {
			URLFetchService fetcher = URLFetchServiceFactory.getURLFetchService();
			String urlStr = ToolString.toUrl(url.trim(), params);
			logger.debug("DELETE : " + urlStr);
			HTTPRequest httpRequest = new HTTPRequest(new URL(urlStr), HTTPMethod.DELETE, FetchOptions.Builder.withDeadline(deadline));
			HTTPResponse response = fetcher.fetch(httpRequest);
			return processResponse(response);
		} catch (Throwable e) {
			retry++;
			if (e instanceof RuntimeException) {
				throw (RuntimeException) e;
			} else if (retry < 3) {
				logger.warn("retrying after " + (System.currentTimeMillis() - start) + " and " + retry + " retries\n" + e.getClass() + e.getMessage());
			} else {
				logger.error(e.getClass() + "\n" + ToolString.stack2string(e));
			}
		}
	}
	return null;
}
 

@Override
public GcsFileMetadata getObjectMetadata(GcsFilename filename, long timeoutMillis)
    throws IOException {
  HTTPRequest req = makeRequest(filename, null, HEAD, timeoutMillis);
  HTTPResponse resp;
  try {
    resp = urlfetch.fetch(req);
  } catch (IOException e) {
    throw createIOException(new HTTPRequestInfo(req), e);
  }
  int responseCode = resp.getResponseCode();
  if (responseCode == 404) {
    return null;
  }
  if (responseCode != 200) {
    throw HttpErrorHandler.error(new HTTPRequestInfo(req), resp);
  }
  return getMetadataFromResponse(
      filename, resp, getLengthFromHeader(resp, X_GOOG_CONTENT_LENGTH));
}
 

public static String getResponseProxyPOST(URL url) throws MalformedURLException, UnsupportedEncodingException, IOException {
	URLFetchService fetcher = URLFetchServiceFactory.getURLFetchService();
	String base64payload = "base64url=" + Base64UrlSafe.encodeServer(url.toString());
	String urlStr = url.toString();
	if (urlStr.contains("?")) {
		base64payload = "base64url=" + Base64UrlSafe.encodeServer(urlStr.substring(0, urlStr.indexOf("?")));
		base64payload += "&base64content=" + Base64UrlSafe.encodeServer(urlStr.substring(urlStr.indexOf("?") + 1));
	} else {
		base64payload = "base64url=" + Base64UrlSafe.encodeServer(urlStr);
	}
	HTTPRequest httpRequest = new HTTPRequest(new URL(HttpClientGAE.POSTPROXY_PHP), HTTPMethod.POST, FetchOptions.Builder.withDeadline(30d).doNotValidateCertificate());
	httpRequest.setPayload(base64payload.getBytes(UTF8));
	HTTPResponse response = fetcher.fetch(httpRequest);
	String processResponse = HttpClientGAE.processResponse(response);
	logger.info("proxying " + url + "\nprocessResponse:" + processResponse);
	return processResponse;
}
 

/**
 * Creates a new instance.
 *
 * @param instanceName the name of the instance to create.
 * @param bootDiskName the name of the disk to create the instance with.
 * @param projectApiKey the project API key.
 * @param accessToken the access token.
 * @param configProperties the configuration properties.
 * @throws MalformedURLException
 * @throws IOException
 * @throws OrchestratorException if the REST API call failed to create instance.
 */
private void createInstance(String instanceName, String bootDiskName, String projectApiKey,
    String accessToken, Map<String, String> configProperties)
    throws MalformedURLException, IOException, OrchestratorException {
  String url = GceApiUtils.composeInstanceApiUrl(
      ConfigProperties.urlPrefixWithProjectAndZone, projectApiKey);
  String payload = createPayload_instance(instanceName, bootDiskName, configProperties);
  logger.info(
      "Calling " + url + " to create instance " + instanceName + "with payload " + payload);
  HTTPResponse httpResponse =
      GceApiUtils.makeHttpRequest(accessToken, url, payload, HTTPMethod.POST);
  int responseCode = httpResponse.getResponseCode();
  if (!(responseCode == 200 || responseCode == 204)) {
    throw new OrchestratorException("Failed to create GCE instance. " + instanceName
        + ". Response code " + responseCode + " Reason: "
        + new String(httpResponse.getContent()));
  }
}
 

@Override
public void composeObject(Iterable<String> source, GcsFilename dest, long timeoutMillis)
    throws IOException {
  StringBuilder xmlContent = new StringBuilder(Iterables.size(source) * 50);
  Escaper escaper = XmlEscapers.xmlContentEscaper();
  xmlContent.append("<ComposeRequest>");
  for (String srcFileName : source) {
    xmlContent.append("<Component><Name>")
        .append(escaper.escape(srcFileName))
        .append("</Name></Component>");
  }
  xmlContent.append("</ComposeRequest>");
  HTTPRequest req = makeRequest(
      dest, COMPOSE_QUERY_STRINGS, PUT, timeoutMillis, xmlContent.toString().getBytes(UTF_8));
  HTTPResponse resp;
  try {
    resp = urlfetch.fetch(req);
  } catch (IOException e) {
    throw createIOException(new HTTPRequestInfo(req), e);
  }
  if (resp.getResponseCode() != 200) {
    throw HttpErrorHandler.error(new HTTPRequestInfo(req), resp);
  }
}
 

/**
 * Checks whether the disk or instance is available.
 *
 * @param accessToken the access token.
 * @param url the URL to check whether the disk/instance has been created.
 * @return true if the disk/instance is available, false otherwise.
 * @throws MalformedURLException
 * @throws IOException
 */
private boolean checkDiskOrInstance(String accessToken, String url)
    throws MalformedURLException, IOException {
  HTTPResponse httpResponse = GceApiUtils.makeHttpRequest(accessToken, url, "", HTTPMethod.GET);
  int responseCode = httpResponse.getResponseCode();
  if (!(responseCode == 200 || responseCode == 204)) {
    logger.fine("Disk/instance not ready. Response code " + responseCode + " Reason: "
        + new String(httpResponse.getContent()));
    return false;
  }
  // Check if the disk/instance is in status "READY".
  String contentStr = new String(httpResponse.getContent());
  JsonParser parser = new JsonParser();
  JsonObject o = (JsonObject) parser.parse(contentStr);
  String status = o.get("status").getAsString();
  if (!status.equals("READY") && !status.equals("RUNNING")) {
    return false;
  }
  return true;
}
 

/**
 * Creates an HTTPRequest with the information passed in.
 *
 * @param accessToken the access token necessary to authorize the request.
 * @param url the url to query.
 * @param payload the payload for the request.
 * @return the created HTTP request.
 * @throws IOException
 */
public static HTTPResponse makeHttpRequest(
    String accessToken, final String url, String payload, HTTPMethod method) throws IOException {

  // Create HTTPRequest and set headers
  HTTPRequest httpRequest = new HTTPRequest(new URL(url.toString()), method);
  httpRequest.addHeader(new HTTPHeader("Authorization", "OAuth " + accessToken));
  httpRequest.addHeader(new HTTPHeader("Host", "www.googleapis.com"));
  httpRequest.addHeader(new HTTPHeader("Content-Length", Integer.toString(payload.length())));
  httpRequest.addHeader(new HTTPHeader("Content-Type", "application/json"));
  httpRequest.addHeader(new HTTPHeader("User-Agent", "google-api-java-client/1.0"));
  httpRequest.setPayload(payload.getBytes());

  URLFetchService fetcher = URLFetchServiceFactory.getURLFetchService();
  HTTPResponse httpResponse = fetcher.fetch(httpRequest);
  return httpResponse;
}
 

@Override
public void copyObject(GcsFilename source, GcsFilename dest, GcsFileOptions fileOptions,
    long timeoutMillis) throws IOException {
  HTTPRequest req = makeRequest(dest, null, PUT, timeoutMillis);
  req.setHeader(new HTTPHeader(X_GOOG_COPY_SOURCE, makePath(source)));
  if (fileOptions != null) {
    req.setHeader(REPLACE_METADATA_HEADER);
    addOptionsHeaders(req, fileOptions);
  }
  HTTPResponse resp;
  try {
    resp = urlfetch.fetch(req);
  } catch (IOException e) {
    throw createIOException(new HTTPRequestInfo(req), e);
  }
  if (resp.getResponseCode() != 200) {
    throw HttpErrorHandler.error(new HTTPRequestInfo(req), resp);
  }
}
 

/**
 * Deletes an instance.
 *
 * @param name the name of the instance to delete.
 * @throws OrchestratorException if delete failed.
 */
private void deleteInstance(String name, String accessToken, String url)
    throws OrchestratorException {
  logger.info("Shutting down instance: " + name);
  HTTPResponse httpResponse;
  try {
    httpResponse = GceApiUtils.makeHttpRequest(accessToken, url, "", HTTPMethod.DELETE);
    int responseCode = httpResponse.getResponseCode();
    if (!(responseCode == 200 || responseCode == 204)) {
      throw new OrchestratorException("Delete Instance failed. Response code " + responseCode
          + " Reason: " + new String(httpResponse.getContent()));
    }
  } catch (IOException e) {
    throw new OrchestratorException(e);
  }
}
 

@Override
public Response finish() throws IOException {
    if (request == null) {
        throw new IllegalStateException("Uploader already closed.");
    }
    request.setPayload(body.toByteArray());
    HTTPResponse response = service.fetch(request);
    Response requestorResponse = toRequestorResponse(response);
    close();
    if (progressListener != null) {
        progressListener.onProgress(request.getPayload().length);
    }
    return requestorResponse;
}
 

/**
 * Deletes a disk.
 *
 * @param diskName the name of the disk to delete.
 * @throws IOException thrown by makeHttpRequest if the Compute Engine API could not be contacted.
 * @throws OrchestratorException if the REST API call failed to delete the disk.
 */
public static void deleteDisk(String diskName, String accessToken, String url)
    throws IOException, OrchestratorException {
  HTTPResponse httpResponse =
      GceApiUtils.makeHttpRequest(accessToken, url, "", HTTPMethod.DELETE);
  int responseCode = httpResponse.getResponseCode();
  if (!(responseCode == 200 || responseCode == 204)) {
    throw new OrchestratorException("Delete Disk failed. Response code " + responseCode
        + " Reason: " + new String(httpResponse.getContent()));
  }
}
 

public static String processResponse(HTTPResponse response) {
	String content = new String(response.getContent(), UTF8);
	if (response.getResponseCode() >= 400) {
		throw new RuntimeException("HttpError:" + response.getResponseCode() + "\n" + content);
	}
	return content;
}
 

public static String getResponsePUT(String url, Map<String, String> params, String json, Map<String, String> headers) {
	int retry = 0;
	while (retry < 3) {
		long start = System.currentTimeMillis();
		try {
			URLFetchService fetcher = URLFetchServiceFactory.getURLFetchService();
			String urlStr = ToolString.toUrl(url.trim(), params);
			logger.debug("PUT : " + urlStr);
			HTTPRequest httpRequest = new HTTPRequest(new URL(urlStr), HTTPMethod.PUT, FetchOptions.Builder.withDeadline(deadline));
			if (headers != null) {
				for (String header : headers.keySet()) {
					httpRequest.addHeader(new HTTPHeader(header, headers.get(header)));
				}
			}
			httpRequest.setPayload(json.getBytes());
			HTTPResponse response = fetcher.fetch(httpRequest);
			return processResponse(response);
		} catch (Throwable e) {
			retry++;
			if (e instanceof RuntimeException) {
				throw (RuntimeException) e;
			} else if (retry < 3) {
				logger.warn("retrying after " + (System.currentTimeMillis() - start) + " and " + retry + " retries\n" + e.getClass() + e.getMessage());
			} else {
				logger.error(e.getClass() + "\n" + ToolString.stack2string(e));
			}
		}
	}
	return null;
}
 

/**
 * Creates a new disk.
 *
 * @param diskName the name of the disk to create.
 * @param accessToken the access token.
 * @param configProperties the configuration properties.
 * @throws MalformedURLException
 * @throws IOException
 * @throws OrchestratorException if the REST API call failed to create disk.
 */
private void createDisk(String diskName, String accessToken, String projectApiKey,
    Map<String, String> configProperties)
    throws MalformedURLException, IOException, OrchestratorException {
  String url =
      GceApiUtils.composeDiskApiUrl(ConfigProperties.urlPrefixWithProjectAndZone, projectApiKey);
  String payload = createPayload_disk(diskName, configProperties);
  HTTPResponse httpResponse =
      GceApiUtils.makeHttpRequest(accessToken, url, payload, HTTPMethod.POST);
  int responseCode = httpResponse.getResponseCode();
  if (!(responseCode == 200 || responseCode == 204)) {
    throw new OrchestratorException("Failed to create Disk " + diskName + ". Response code "
        + responseCode + " Reason: " + new String(httpResponse.getContent()));
  }
}
 

static String describeRequestAndResponse(HTTPRequestInfo req, HTTPResponse resp) {
  StringBuilder b = new StringBuilder(256).append("Request: ");
  req.appendToString(b);
  b.append("\nResponse: ");
  appendResponse(resp, b);
  return b.toString();
}
 

@Test(expected = IOException.class)
public void fetchNonExistentLocalAppThrowsException() throws Exception {
    URL selfURL = new URL("BOGUS-" + appUrlBase + "/");
    FetchOptions fetchOptions = FetchOptions.Builder.withDefaults()
        .doNotFollowRedirects()
        .setDeadline(10.0);
    HTTPRequest httpRequest = new HTTPRequest(selfURL, HTTPMethod.GET, fetchOptions);
    URLFetchService urlFetchService = URLFetchServiceFactory.getURLFetchService();
    HTTPResponse httpResponse = urlFetchService.fetch(httpRequest);
    fail("expected exception, got " + httpResponse.getResponseCode());
}
 

@Test
public void testDoNotFollowRedirects() throws Exception {
    final URL redirect = getUrl("redirect");
    FetchOptions options = buildFetchOptions();
    options.doNotFollowRedirects();
    testOptions(redirect, options, new ResponseHandler() {
        public void handle(HTTPResponse response) throws Exception {
            Assert.assertEquals(302, response.getResponseCode());
        }
    });
}
 

@Test
public void testFollowRedirects() throws Exception {
    final URL redirect = getUrl("redirect");
    FetchOptions options = buildFetchOptions();
    options.followRedirects();
    testOptions(redirect, options, new ResponseHandler() {
        public void handle(HTTPResponse response) throws Exception {
            URL finalURL = response.getFinalUrl();
            Assert.assertEquals(getUrl(""), finalURL);
        }
    });
}
 

/** Gets all headers with the name {@code headerName}, case-insensitive. */
private static Iterable<HTTPHeader> getHeaders(HTTPResponse resp, String headerName) {
  final String lowercaseHeaderName = headerName.toLowerCase();
  return Iterables.filter(resp.getHeadersUncombined(), new Predicate<HTTPHeader>() {
    @Override public boolean apply(HTTPHeader header) {
      return header.getName().toLowerCase().equals(lowercaseHeaderName);
    }
  });
}
 

static void appendResponse(HTTPResponse resp, StringBuilder b) {
  byte[] content = resp.getContent();
  b.append(resp.getResponseCode()).append(" with ").append(content == null ? 0 : content.length);
  b.append(" bytes of content");
  for (HTTPHeader h : resp.getHeadersUncombined()) {
    b.append('\n').append(h.getName()).append(": ").append(h.getValue());
  }
  b.append('\n').append(content == null ? "" : new String(content, UTF_8)).append('\n');
}