org.apache.http.client.RedirectException Java Examples

The following examples show how to use org.apache.http.client.RedirectException. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: URLRespectsRobotsTest.java    From BUbiNG with Apache License 2.0 4 votes vote down vote up 
@Test
public void testRedirectSync() throws Exception {
	proxy = new SimpleFixedHttpProxy();
	URI robotsURL0 = URI.create("http://foo.bar/robots.txt");
	URI robotsURL1 = URI.create("http://foo.bar/fubar/robots.txt");

	proxy.addNon200(robotsURL0, "HTTP/1.1 301 Moved Permanently\nLocation: " + robotsURL1 + "\n", "");
	proxy.add200(robotsURL1, "",
			"# goodguy can do anything\n" +
			"User-agent: goodguy\n" +
			"Disallow:\n\n" +
			"# every other guy can do nothing\n" +
			"User-agent: *\n" +
			"Disallow: /\n"
	);
	URI url = URI.create("http://foo.bar/goo/zoo.html"); // Disallowed
	proxy.add200(url, "", "Should not be crawled...");

	proxy.addNon200(URI.create("http://too.many/robots.txt"), "HTTP/1.1 301 Moved Permanently\nLocation: http://too.many/0\n", "");
	for(int i = 0; i < 5; i++) proxy.addNon200(URI.create("http://too.many/" + i), "HTTP/1.1 301 Moved Permanently\nLocation: http://too.many/" + (i + 1) + "\n", "");

	proxy.start();

	HttpClient httpClient = FetchDataTest.getHttpClient(new HttpHost("localhost", proxy.port()), true);

	FetchData fetchData = new FetchData(Helpers.getTestConfiguration(this));

	fetchData.fetch(URI.create(BURL.schemeAndAuthority(url) + "/robots.txt"), httpClient, null, null, true);
	char[][] filter = URLRespectsRobots.parseRobotsResponse(fetchData, "goodGuy");
	assertTrue(URLRespectsRobots.apply(filter, url));
	filter = URLRespectsRobots.parseRobotsResponse(fetchData, "badGuy");
	assertFalse(URLRespectsRobots.apply(filter, url));

	filter = URLRespectsRobots.parseRobotsResponse(fetchData, "goodGuy foo");
	assertTrue(URLRespectsRobots.apply(filter, url));
	filter = URLRespectsRobots.parseRobotsResponse(fetchData, "badGuy foo");
	assertFalse(URLRespectsRobots.apply(filter, url));

	fetchData = new FetchData(Helpers.getTestConfiguration(this));
	fetchData.fetch(URI.create("http://too.many/robots.txt"), httpClient, null, null, true);
	assertTrue(fetchData.exception instanceof RedirectException);

	fetchData.close();
}