Java Code Examples for com.gargoylesoftware.htmlunit.WebRequest#setHttpMethod()

/**
 * Checks that CSRF validation works if token sent as header instead of
 * form field.
 *
 * @throws Exception an error occurs or validation fails.
 */
@Test
public void testFormHeaderOk() throws Exception {
    HtmlPage page1 = webClient.getPage(webUrl + "resources/csrf");

    // Check response and CSRF header
    WebResponse res = page1.getWebResponse();
    assertEquals(Response.Status.OK.getStatusCode(), res.getStatusCode());
    assertNotNull(res.getResponseHeaderValue(CSRF_HEADER));

    WebRequest req = new WebRequest(new URL(webUrl + "resources/csrf"));
    req.setHttpMethod(HttpMethod.POST);
    req.setAdditionalHeader(CSRF_HEADER, res.getResponseHeaderValue(CSRF_HEADER));
    res = webClient.loadWebResponse(req);
    assertEquals(Response.Status.OK.getStatusCode(), res.getStatusCode());
}
 

/**
 * Checks that CSRF validation works if token sent as header instead of
 * form field.
 *
 * @throws Exception an error occurs or validation fails.
 */
@Test
public void testFormHeaderOk() throws Exception {
    HtmlPage page1 = webClient.getPage(webUrl + "resources/csrf");

    // Check response and CSRF header
    WebResponse res = page1.getWebResponse();
    assertEquals(Response.Status.OK.getStatusCode(), res.getStatusCode());
    assertNotNull(res.getResponseHeaderValue(CSRF_HEADER));

    WebRequest req = new WebRequest(new URL(webUrl + "resources/csrf"));
    req.setHttpMethod(HttpMethod.POST);
    req.setAdditionalHeader(CSRF_HEADER, res.getResponseHeaderValue(CSRF_HEADER));
    res = webClient.loadWebResponse(req);
    assertEquals(Response.Status.OK.getStatusCode(), res.getStatusCode());
}
 

/**
 * Checks that regular JAX-RS resource method is not protected for CSRF.
 *
 * @throws Exception an error occurs or validation fails.
 */
@Test
public void testJaxrsOk() throws Exception {
    WebRequest req = new WebRequest(new URL(webUrl + "resources/csrf/jaxrs"));
    req.setHttpMethod(HttpMethod.POST);
    WebResponse res = webClient.loadWebResponse(req);
    assertEquals(Response.Status.OK.getStatusCode(), res.getStatusCode());
}
 

/**
 * HtmlUnit을 이용한 HTML 코드 파싱.
 *
 * @param eachArchiveAddress 실제 만화가 담긴 아카이브 주소
 * @return 성공 시 html 코드를 리턴
 */
private String getHtmlPageHtmlUnit(String eachArchiveAddress) throws Exception {
	/* 필수! 로그 메세지 출력 안함 -> HtmlUnit 이용시 Verbose한 로그들이 너무 많아서 다 끔 */
	java.util.logging.Logger.getLogger("com.gargoylesoftware").setLevel(Level.OFF);
	System.setProperty("org.apache.commons.logging.Log", "org.apache.commons.logging.impl.NoOpLog");

	print.info("일반 연결 시도중...\n");

	WebClient webClient = new WebClient();
	webClient.getOptions().setRedirectEnabled(true);

	WebRequest req = new WebRequest(new URL(eachArchiveAddress));
	req.setHttpMethod(HttpMethod.POST);
	req.setAdditionalHeader("User-Agent", UserAgent.getUserAgent());
	req.setAdditionalHeader("Accept-Encoding", "gzip"); //20171126 gzip 추가
	req.getRequestParameters().add(new NameValuePair("pass", PASSWORD)); //비밀번호 post 방식 전송

	HtmlPage page = webClient.getPage(req);

	//Html코드를 포함한 페이지 소스코드가 담길 스트링
	String pageSource = page.asXml();

	/** 여기도 페이지 파싱 실패 시 검증하는 코드 들어가야 됨 **/

	webClient.close();
	print.info("일반 연결 성공\n");
	return pageSource;
}
 

/**
 * Checks that regular JAX-RS resource method is not protected for CSRF.
 *
 * @throws Exception an error occurs or validation fails.
 */
@Test
public void testJaxrsOk() throws Exception {
    WebRequest req = new WebRequest(new URL(webUrl + "resources/csrf/jaxrs"));
    req.setHttpMethod(HttpMethod.POST);
    WebResponse res = webClient.loadWebResponse(req);
    assertEquals(Response.Status.OK.getStatusCode(), res.getStatusCode());
}
 

private Page post(String relative, String userName) throws Exception {
    final JenkinsRule.WebClient client;
    if (userName != null) {
        client = j.createWebClient().login(userName);
    } else {
        client = j.createWebClient();
    }

    final WebRequest request = new WebRequest(new URL(client.getContextPath() + relative), client.getBrowserVersion().getHtmlAcceptHeader());
    request.setHttpMethod(HttpMethod.POST);
    request.setRequestParameters(Arrays.asList(new NameValuePair(Functions.getCrumbRequestField(), Functions.getCrumb(null))));
    return client.getPage(request);
}
 

@Before
public void setup() throws Exception {
	webRequest = new WebRequest(new URL("http://example.com:80/test/this/here"));
	webRequest.setHttpMethod(HttpMethod.GET);
	requestBuilder = new HtmlUnitRequestBuilder(sessions, webClient, webRequest);
}
 

@Before
public void setup() throws Exception {
	webRequest = new WebRequest(new URL("http://example.com:80/test/this/here"));
	webRequest.setHttpMethod(HttpMethod.GET);
	requestBuilder = new HtmlUnitRequestBuilder(sessions, webClient, webRequest);
}
 

@Override
public Document load(PageRequest pageRequest) {
    if (!UrlUtil.isUrl(pageRequest.getUrl())) {
        return null;
    }

    WebClient webClient = new WebClient();
    try {
        WebRequest webRequest = new WebRequest(new URL(pageRequest.getUrl()));

        // 请求设置
        webClient.getOptions().setUseInsecureSSL(true);
        webClient.getOptions().setJavaScriptEnabled(true);
        webClient.getOptions().setCssEnabled(false);
        webClient.getOptions().setThrowExceptionOnScriptError(false);
        webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
        webClient.getOptions().setDoNotTrackEnabled(false);
        webClient.getOptions().setUseInsecureSSL(!pageRequest.isValidateTLSCertificates());

        if (pageRequest.getParamMap() != null && !pageRequest.getParamMap().isEmpty()) {
            for (Map.Entry<String, String> paramItem : pageRequest.getParamMap().entrySet()) {
                webRequest.getRequestParameters().add(new NameValuePair(paramItem.getKey(), paramItem.getValue()));
            }
        }
        if (pageRequest.getCookieMap() != null && !pageRequest.getCookieMap().isEmpty()) {
            webClient.getCookieManager().setCookiesEnabled(true);
            for (Map.Entry<String, String> cookieItem : pageRequest.getCookieMap().entrySet()) {
                webClient.getCookieManager().addCookie(new Cookie("", cookieItem.getKey(), cookieItem.getValue()));
            }
        }
        if (pageRequest.getHeaderMap() != null && !pageRequest.getHeaderMap().isEmpty()) {
            webRequest.setAdditionalHeaders(pageRequest.getHeaderMap());
        }
        if (pageRequest.getUserAgent() != null) {
            webRequest.setAdditionalHeader("User-Agent", pageRequest.getUserAgent());
        }
        if (pageRequest.getReferrer() != null) {
            webRequest.setAdditionalHeader("Referer", pageRequest.getReferrer());
        }

        webClient.getOptions().setTimeout(pageRequest.getTimeoutMillis());
        webClient.setJavaScriptTimeout(pageRequest.getTimeoutMillis());
        webClient.waitForBackgroundJavaScript(pageRequest.getTimeoutMillis());

        // 代理
        if (pageRequest.getProxy() != null) {
            InetSocketAddress address = (InetSocketAddress) pageRequest.getProxy().address();
            boolean isSocks = pageRequest.getProxy().type() == Proxy.Type.SOCKS;
            webClient.getOptions().setProxyConfig(new ProxyConfig(address.getHostName(), address.getPort(), isSocks));
        }

        // 发出请求
        if (pageRequest.isIfPost()) {
            webRequest.setHttpMethod(HttpMethod.POST);
        } else {
            webRequest.setHttpMethod(HttpMethod.GET);
        }
        HtmlPage page = webClient.getPage(webRequest);

        String pageAsXml = page.asXml();
        if (pageAsXml != null) {
            Document html = Jsoup.parse(pageAsXml);
            return html;
        }
    } catch (IOException e) {
        logger.error(e.getMessage(), e);
    } finally {
        if (webClient != null) {
            webClient.close();
        }
    }
    return null;
}
 

public HttpResponse download(HttpRequest request, int timeout) throws DownloadException {
	try {
		URL url = new URL(request.getUrl());
		WebRequest webRequest = new WebRequest(url);
		webRequest.setHttpMethod(HttpMethod.GET);
		if(request instanceof HttpPostRequest) {//post
			HttpPostRequest post = (HttpPostRequest)request;
			webRequest.setHttpMethod(HttpMethod.POST);
			List<NameValuePair> requestParameters = new ArrayList<NameValuePair>();
			for(Map.Entry<String, Object> entry : post.getFields().entrySet()) {
				NameValuePair nvp = new NameValuePair(entry.getKey(), entry.getValue().toString());
				requestParameters.add(nvp);
			}
			webRequest.setRequestParameters(requestParameters);	
		}
		//header
		boolean isMobile = SpiderThreadLocal.get().getEngine().isMobile();
		webRequest.setAdditionalHeader("User-Agent", UserAgent.getUserAgent(isMobile));
		webRequest.setAdditionalHeaders(request.getHeaders());
		//proxy
		HttpHost proxy = Proxys.getProxy();
		if(proxy != null) {
			webRequest.setProxyHost(proxy.getHostName());
			webRequest.setProxyPort(proxy.getPort());
		}
		//timeout
		this.webClient.getOptions().setTimeout(timeout);
		//request,response
		webClient.getPage(webRequest);
		HtmlPage page = webClient.getPage(request.getUrl());
		HttpResponse resp = new HttpResponse();
		WebResponse webResponse = page.getWebResponse();
		int status = webResponse.getStatusCode();
		resp.setStatus(status);
		if(status == 302 || status == 301) {
			String redirectUrl = webResponse.getResponseHeaderValue("Location");
			resp.setContent(UrlUtils.relative2Absolute(request.getUrl(), redirectUrl));
		} else if(status == 200) {
			String content = page.asXml();
			resp.setContent(content);
			resp.setRaw(webResponse.getContentAsStream());
			String contentType = webResponse.getContentType();
			resp.setContentType(contentType);
			String charset = getCharset(request.getCharset(), contentType);
			resp.setCharset(charset);
		} else {
			throw new DownloadException("ERROR : " + status);
		}
		return resp;
	} catch(Exception ex) {
		throw new DownloadException(ex);
	}
}
 

@Before
public void setUp() throws Exception {
	webRequest = new WebRequest(new URL("http://example.com:80/test/this/here"));
	webRequest.setHttpMethod(HttpMethod.GET);
	requestBuilder = new HtmlUnitRequestBuilder(sessions, webClient, webRequest);
}