com.google.appengine.api.urlfetch.URLFetchService Java Examples

/**
 * Creates an HTTPRequest with the information passed in.
 *
 * @param accessToken the access token necessary to authorize the request.
 * @param url the url to query.
 * @param payload the payload for the request.
 * @return the created HTTP request.
 * @throws IOException
 */
public static HTTPResponse makeHttpRequest(
    String accessToken, final String url, String payload, HTTPMethod method) throws IOException {

  // Create HTTPRequest and set headers
  HTTPRequest httpRequest = new HTTPRequest(new URL(url.toString()), method);
  httpRequest.addHeader(new HTTPHeader("Authorization", "OAuth " + accessToken));
  httpRequest.addHeader(new HTTPHeader("Host", "www.googleapis.com"));
  httpRequest.addHeader(new HTTPHeader("Content-Length", Integer.toString(payload.length())));
  httpRequest.addHeader(new HTTPHeader("Content-Type", "application/json"));
  httpRequest.addHeader(new HTTPHeader("User-Agent", "google-api-java-client/1.0"));
  httpRequest.setPayload(payload.getBytes());

  URLFetchService fetcher = URLFetchServiceFactory.getURLFetchService();
  HTTPResponse httpResponse = fetcher.fetch(httpRequest);
  return httpResponse;
}
 

@Test
public void testAsyncOps() throws Exception {
    URLFetchService service = URLFetchServiceFactory.getURLFetchService();

    URL adminConsole = findAvailableUrl(URLS);
    Future<HTTPResponse> response = service.fetchAsync(adminConsole);
    printResponse(response.get(5, TimeUnit.SECONDS));

    response = service.fetchAsync(new HTTPRequest(adminConsole));
    printResponse(response.get(5, TimeUnit.SECONDS));

    URL jbossOrg = new URL("http://www.jboss.org");
    if (available(jbossOrg)) {
        response = service.fetchAsync(jbossOrg);
        printResponse(response.get(30, TimeUnit.SECONDS));
    }

    sync(5000L); // wait a bit for async to finish
}
 

public static String getResponseDELETE(String url, Map<String, String> params, Map<String, String> headers) {
	int retry = 0;
	while (retry < 3) {
		long start = System.currentTimeMillis();
		try {
			URLFetchService fetcher = URLFetchServiceFactory.getURLFetchService();
			String urlStr = ToolString.toUrl(url.trim(), params);
			logger.debug("DELETE : " + urlStr);
			HTTPRequest httpRequest = new HTTPRequest(new URL(urlStr), HTTPMethod.DELETE, FetchOptions.Builder.withDeadline(deadline));
			HTTPResponse response = fetcher.fetch(httpRequest);
			return processResponse(response);
		} catch (Throwable e) {
			retry++;
			if (e instanceof RuntimeException) {
				throw (RuntimeException) e;
			} else if (retry < 3) {
				logger.warn("retrying after " + (System.currentTimeMillis() - start) + " and " + retry + " retries\n" + e.getClass() + e.getMessage());
			} else {
				logger.error(e.getClass() + "\n" + ToolString.stack2string(e));
			}
		}
	}
	return null;
}
 

/**
 * @param request HTTP request to execute.
 * @param client The client used to execute the request.
 * @param responseClass Model class to unmarshal JSON body content.
 * @param fieldNamingPolicy FieldNamingPolicy for unmarshaling JSON.
 * @param errorTimeOut Number of milliseconds to re-send erroring requests.
 * @param maxRetries Number of times allowed to re-send erroring requests.
 */
public GaePendingResult(
    HTTPRequest request,
    URLFetchService client,
    Class<R> responseClass,
    FieldNamingPolicy fieldNamingPolicy,
    long errorTimeOut,
    Integer maxRetries,
    ExceptionsAllowedToRetry exceptionsAllowedToRetry,
    RequestMetrics metrics) {
  this.request = request;
  this.client = client;
  this.responseClass = responseClass;
  this.fieldNamingPolicy = fieldNamingPolicy;
  this.errorTimeOut = errorTimeOut;
  this.maxRetries = maxRetries;
  this.exceptionsAllowedToRetry = exceptionsAllowedToRetry;
  this.metrics = metrics;

  metrics.startNetwork();
  this.call = client.fetchAsync(request);
}
 

public void doGet(HttpServletRequest req, HttpServletResponse res)
    throws IOException, ServletException {
  String url = req.getParameter("url");
  String deadlineSecs = req.getParameter("deadline");
  URLFetchService service = URLFetchServiceFactory.getURLFetchService();
  HTTPRequest fetchReq = new HTTPRequest(new URL(url));
  if (deadlineSecs != null) {
    fetchReq.getFetchOptions().setDeadline(Double.valueOf(deadlineSecs));
  }
  HTTPResponse fetchRes = service.fetch(fetchReq);
  for (HTTPHeader header : fetchRes.getHeaders()) {
    res.addHeader(header.getName(), header.getValue());
  }
  if (fetchRes.getResponseCode() == 200) {
    res.getOutputStream().write(fetchRes.getContent());
  } else {
    res.sendError(fetchRes.getResponseCode(), "Error while fetching");
  }
}
 

@Override
public LowLevelHttpResponse execute() throws IOException {
  // write content
  if (getStreamingContent() != null) {
    String contentType = getContentType();
    if (contentType != null) {
      addHeader("Content-Type", contentType);
    }
    String contentEncoding = getContentEncoding();
    if (contentEncoding != null) {
      addHeader("Content-Encoding", contentEncoding);
    }
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    getStreamingContent().writeTo(out);
    byte[] payload = out.toByteArray();
    if (payload.length != 0) {
      request.setPayload(payload);
    }
  }
  // connect
  URLFetchService service = URLFetchServiceFactory.getURLFetchService();
  HTTPResponse response = service.fetch(request);
  return new UrlFetchResponse(response);
}
 

public static String getResponseProxyPOST(URL url) throws MalformedURLException, UnsupportedEncodingException, IOException {
	URLFetchService fetcher = URLFetchServiceFactory.getURLFetchService();
	String base64payload = "base64url=" + Base64UrlSafe.encodeServer(url.toString());
	String urlStr = url.toString();
	if (urlStr.contains("?")) {
		base64payload = "base64url=" + Base64UrlSafe.encodeServer(urlStr.substring(0, urlStr.indexOf("?")));
		base64payload += "&base64content=" + Base64UrlSafe.encodeServer(urlStr.substring(urlStr.indexOf("?") + 1));
	} else {
		base64payload = "base64url=" + Base64UrlSafe.encodeServer(urlStr);
	}
	HTTPRequest httpRequest = new HTTPRequest(new URL(HttpClientGAE.POSTPROXY_PHP), HTTPMethod.POST, FetchOptions.Builder.withDeadline(30d).doNotValidateCertificate());
	httpRequest.setPayload(base64payload.getBytes(UTF8));
	HTTPResponse response = fetcher.fetch(httpRequest);
	String processResponse = HttpClientGAE.processResponse(response);
	logger.info("proxying " + url + "\nprocessResponse:" + processResponse);
	return processResponse;
}
 

@Override
public void doGet(HttpServletRequest req, HttpServletResponse resp)
    throws IOException, ServletException {
  String trackingId = System.getenv("GA_TRACKING_ID");
  URIBuilder builder = new URIBuilder();
  builder
      .setScheme("http")
      .setHost("www.google-analytics.com")
      .setPath("/collect")
      .addParameter("v", "1") // API Version.
      .addParameter("tid", trackingId) // Tracking ID / Property ID.
      // Anonymous Client Identifier. Ideally, this should be a UUID that
      // is associated with particular user, device, or browser instance.
      .addParameter("cid", "555")
      .addParameter("t", "event") // Event hit type.
      .addParameter("ec", "example") // Event category.
      .addParameter("ea", "test action"); // Event action.
  URI uri = null;
  try {
    uri = builder.build();
  } catch (URISyntaxException e) {
    throw new ServletException("Problem building URI", e);
  }
  URLFetchService fetcher = URLFetchServiceFactory.getURLFetchService();
  URL url = uri.toURL();
  fetcher.fetch(url);
  resp.getWriter().println("Event tracked.");
}
 

public static String getResponseGET(String url, Map<String, String> params, Map<String, String> headers) {
	int retry = 0;
	while (retry < 3) {
		long start = System.currentTimeMillis();
		try {
			URLFetchService fetcher = URLFetchServiceFactory.getURLFetchService();
			String urlStr = ToolString.toUrl(url.trim(), params);
			HTTPRequest httpRequest = new HTTPRequest(new URL(urlStr), HTTPMethod.GET, FetchOptions.Builder.withDeadline(deadline));
			if (headers != null) {
				for (String name : headers.keySet()) {
					httpRequest.addHeader(new HTTPHeader(name, headers.get(name)));
				}
			}
			return processResponse(fetcher.fetch(httpRequest));
		} catch (Throwable e) {
			retry++;
			if (e instanceof RuntimeException) {
				throw (RuntimeException) e;
			} else if (retry < 3) {
				logger.warn("retrying after " + (System.currentTimeMillis() - start) + " and " + retry + " retries\n" + e.getClass() + e.getMessage());
			} else {
				logger.error(e.getClass() + "\n" + ToolString.stack2string(e));
			}
		}
	}
	return null;
}
 

public static String getResponsePUT(String url, Map<String, String> params, String json, Map<String, String> headers) {
	int retry = 0;
	while (retry < 3) {
		long start = System.currentTimeMillis();
		try {
			URLFetchService fetcher = URLFetchServiceFactory.getURLFetchService();
			String urlStr = ToolString.toUrl(url.trim(), params);
			logger.debug("PUT : " + urlStr);
			HTTPRequest httpRequest = new HTTPRequest(new URL(urlStr), HTTPMethod.PUT, FetchOptions.Builder.withDeadline(deadline));
			if (headers != null) {
				for (String header : headers.keySet()) {
					httpRequest.addHeader(new HTTPHeader(header, headers.get(header)));
				}
			}
			httpRequest.setPayload(json.getBytes());
			HTTPResponse response = fetcher.fetch(httpRequest);
			return processResponse(response);
		} catch (Throwable e) {
			retry++;
			if (e instanceof RuntimeException) {
				throw (RuntimeException) e;
			} else if (retry < 3) {
				logger.warn("retrying after " + (System.currentTimeMillis() - start) + " and " + retry + " retries\n" + e.getClass() + e.getMessage());
			} else {
				logger.error(e.getClass() + "\n" + ToolString.stack2string(e));
			}
		}
	}
	return null;
}
 

@Test
public void testAuthIsRetried() throws IOException, InterruptedException, ExecutionException {
  URLFetchService urlFetchService = mock(URLFetchService.class, RETURNS_MOCKS);
  FailingFetchService failingFetchService = new FailingFetchService(urlFetchService, 1);
  failingFetchService.fetch(mock(HTTPRequest.class));

  failingFetchService = new FailingFetchService(urlFetchService, 1);
  failingFetchService.fetchAsync(mock(HTTPRequest.class)).get();
}
 

@Test(expected = IOException.class)
public void fetchNonExistentLocalAppThrowsException() throws Exception {
    URL selfURL = new URL("BOGUS-" + appUrlBase + "/");
    FetchOptions fetchOptions = FetchOptions.Builder.withDefaults()
        .doNotFollowRedirects()
        .setDeadline(10.0);
    HTTPRequest httpRequest = new HTTPRequest(selfURL, HTTPMethod.GET, fetchOptions);
    URLFetchService urlFetchService = URLFetchServiceFactory.getURLFetchService();
    HTTPResponse httpResponse = urlFetchService.fetch(httpRequest);
    fail("expected exception, got " + httpResponse.getResponseCode());
}
 

@Test
public void testBasicOps() throws Exception {
    URLFetchService service = URLFetchServiceFactory.getURLFetchService();

    URL adminConsole = findAvailableUrl(URLS);
    HTTPResponse response = service.fetch(adminConsole);
    printResponse(response);

    URL jbossOrg = new URL("http://www.jboss.org");
    if (available(jbossOrg)) {
        response = service.fetch(jbossOrg);
        printResponse(response);
    }
}
 

@Test
public void testPayload() throws Exception {
    URLFetchService service = URLFetchServiceFactory.getURLFetchService();

    URL url = getFetchUrl();

    HTTPRequest req = new HTTPRequest(url, HTTPMethod.POST);
    req.setHeader(new HTTPHeader("Content-Type", "application/octet-stream"));
    req.setPayload("Tralala".getBytes(UTF_8));

    HTTPResponse response = service.fetch(req);
    String content = new String(response.getContent());
    Assert.assertEquals("Hopsasa", content);
}
 

@Test
public void testFollowRedirectsExternal() throws Exception {
    final URL redirectUrl = new URL("http://google.com/");
    final String expectedDestinationURLPrefix = "http://www.google.";

    FetchOptions options = FetchOptions.Builder.followRedirects();

    HTTPRequest request = new HTTPRequest(redirectUrl, HTTPMethod.GET, options);
    URLFetchService service = URLFetchServiceFactory.getURLFetchService();
    HTTPResponse response = service.fetch(request);
    String destinationUrl = response.getFinalUrl().toString();
    assertTrue("Did not get redirected.", destinationUrl.startsWith(expectedDestinationURLPrefix));
}
 

public FetchServiceUploader(URLFetchService service, HTTPRequest request, ByteArrayOutputStream body) {
    this.service = service;
    this.request = request;
    this.body = body;
}
 

public URLFetchService getService() {
    return service;
}
 

protected void testOptions(URL url, HTTPMethod method, FetchOptions options, ResponseHandler handler) throws Exception {
    HTTPRequest request = new HTTPRequest(url, method, options);
    URLFetchService service = URLFetchServiceFactory.getURLFetchService();
    HTTPResponse response = service.fetch(request);
    handler.handle(response);
}
 

public GoogleAppEngineRequestor(FetchOptions options, URLFetchService service) {
    if (options == null) throw new NullPointerException("options");
    if (service == null) throw new NullPointerException("service");
    this.options = options;
    this.service = service;
}
 

@Provides
static URLFetchService provideURLFetchService() {
  return fetchService;
}
 

protected String fetchUrl(String url, int expectedResponse) throws IOException {
    URLFetchService urlFetchService = URLFetchServiceFactory.getURLFetchService();
    HTTPResponse httpResponse = urlFetchService.fetch(new URL(url));
    assertEquals(url, expectedResponse, httpResponse.getResponseCode());
    return new String(httpResponse.getContent());
}
 

FailingFetchService(URLFetchService urlFetch, int numToFail) {
  super(urlFetch);
  this.numToFail = numToFail;
}
 

AppIdentityOAuthURLFetchService(URLFetchService urlFetch, List<String> oauthScopes) {
  super(urlFetch);
  this.oauthScopes = ImmutableList.copyOf(oauthScopes);
  this.accessTokenProvider = createAccessTokenProvider();
}
 

AbstractOAuthURLFetchService(URLFetchService urlFetch) {
  this.urlFetch = checkNotNull(urlFetch);
}
 

@Inject
public AppEngineHttpFetcher(URLFetchService fetchService) {
  this.fetchService = fetchService;
}
 

@Provides @Singleton
public URLFetchService providerUrlFetchService() {
  return URLFetchServiceFactory.getURLFetchService();
}