Java Code Examples for com.gargoylesoftware.htmlunit.WebClient#setJavaScriptTimeout()

The following examples show how to use com.gargoylesoftware.htmlunit.WebClient#setJavaScriptTimeout() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: JavascriptErrorListenerTest.java From htmlunit with Apache License 2.0

6 votes

@Test(timeout = 10_000)
public void listenForTimeoutError() throws Exception {
    final WebClient webClient = getWebClient();
    webClient.getOptions().setThrowExceptionOnScriptError(false);
    final CollectingJavaScriptErrorListener javaScriptErrorListener = new CollectingJavaScriptErrorListener();
    webClient.setJavaScriptErrorListener(javaScriptErrorListener);
    webClient.setJavaScriptTimeout(100);

    final String html = "<html><head><title>Throw JavaScript Timeout Error</title>\n"
        + "<script>while(1) {}</script></head>\n"
        + "<body></body></html>";

    loadPage(html);

    assertEquals("", javaScriptErrorListener.getWarnings());
    assertEquals("", javaScriptErrorListener.getScriptExceptions());
    assertEquals("", javaScriptErrorListener.getLoadScriptErrors());
    assertEquals("", javaScriptErrorListener.getMalformedScriptURLErrors());
    assertEquals("Timeout allowed: 100", javaScriptErrorListener.getTimeoutErrors());
}

Example 2

Source File: HtmlUnitFetcher.java From sparkler with Apache License 2.0

5 votes

@Override
public void init(JobContext context, String pluginId) throws SparklerException {
    super.init(context, pluginId);
    //TODO: get timeouts from configurations
    driver = new WebClient(BrowserVersion.BEST_SUPPORTED);
    driver.setJavaScriptTimeout(DEFAULT_JS_TIMEOUT);
    WebClientOptions options = driver.getOptions();
    options.setCssEnabled(false);
    options.setAppletEnabled(false);
    options.setDownloadImages(false);
    options.setJavaScriptEnabled(true);
    options.setTimeout(DEFAULT_TIMEOUT);
    options.setUseInsecureSSL(true);
    options.setPopupBlockerEnabled(true);
    options.setDoNotTrackEnabled(true);
    options.setGeolocationEnabled(false);
    options.setHistorySizeLimit(2);
    options.setPrintContentOnFailingStatusCode(false);
    options.setThrowExceptionOnScriptError(false);
    options.setThrowExceptionOnFailingStatusCode(false);
    if (this.httpHeaders != null && !this.httpHeaders.isEmpty()) {
        LOG.info("Found {} headers", this.httpHeaders.size());
        this.httpHeaders.forEach((name, val) -> driver.addRequestHeader(name, val));
    } else {
        LOG.info("No user headers found");
    }
}

Example 3

Source File: JavascriptErrorListenerTest.java From htmlunit with Apache License 2.0

4 votes

/**
 * Test for running without a JavaScript error listener.
 *
 * @throws Exception if the test fails
 */
@Test
public void nullJavaScriptErrorListener() throws Exception {
    final WebClient webClient = getWebClient();
    webClient.getOptions().setThrowExceptionOnScriptError(false);
    webClient.setJavaScriptErrorListener(null);
    final MockWebConnection webConnection = new MockWebConnection();
    final String errorContent = "<html><head><title>ERROR 500</title></head><body></body></html>";
    final List<NameValuePair> emptyList = Collections.emptyList();
    webConnection.setResponse(URL_SECOND, errorContent, 500, "BOOM", MimeType.TEXT_HTML, emptyList);

    // test script exception
    String content = "<html><head><title>Throw JavaScript Error</title>\n"
        + "<script>unknown.foo();</script></head>\n"
        + "<body></body></html>";
    webConnection.setResponse(URL_FIRST, content);
    webClient.setWebConnection(webConnection);
    webClient.getPage(URL_FIRST);

    // test load script error
    content = "<html><head><title>Throw JavaScript Error</title>\n"
        + "<script src='" + URL_SECOND + "' type='text/javascript'></script></head>\n"
        + "<body></body></html>";
    webConnection.setResponse(URL_FIRST, content);
    try {
        webClient.getPage(URL_FIRST);
        fail("FailingHttpStatusCodeException expected");
    }
    catch (final FailingHttpStatusCodeException e) {
        // expected
    }

    // test malformed script url error
    content = "<html><head><title>Throw JavaScript Error</title>\n"
        + "<script src='unknown://nowhere' type='text/javascript'></script></head>\n"
        + "<body></body></html>";
    webConnection.setResponse(URL_FIRST, content);
    webClient.getPage(URL_FIRST);

    // test timeout error
    webClient.setJavaScriptTimeout(100);

    content = "<html><head><title>Throw JavaScript Timeout Error</title>\n"
        + "<script>while(1) {}</script></head>\n"
        + "<body></body></html>";
    webConnection.setResponse(URL_FIRST, content);
    webClient.getPage(URL_FIRST);
}

Example 4

Source File: HtmlUnitPageLoader.java From xxl-crawler with GNU General Public License v3.0

4 votes

@Override
public Document load(PageRequest pageRequest) {
    if (!UrlUtil.isUrl(pageRequest.getUrl())) {
        return null;
    }

    WebClient webClient = new WebClient();
    try {
        WebRequest webRequest = new WebRequest(new URL(pageRequest.getUrl()));

        // 请求设置
        webClient.getOptions().setUseInsecureSSL(true);
        webClient.getOptions().setJavaScriptEnabled(true);
        webClient.getOptions().setCssEnabled(false);
        webClient.getOptions().setThrowExceptionOnScriptError(false);
        webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
        webClient.getOptions().setDoNotTrackEnabled(false);
        webClient.getOptions().setUseInsecureSSL(!pageRequest.isValidateTLSCertificates());

        if (pageRequest.getParamMap() != null && !pageRequest.getParamMap().isEmpty()) {
            for (Map.Entry<String, String> paramItem : pageRequest.getParamMap().entrySet()) {
                webRequest.getRequestParameters().add(new NameValuePair(paramItem.getKey(), paramItem.getValue()));
            }
        }
        if (pageRequest.getCookieMap() != null && !pageRequest.getCookieMap().isEmpty()) {
            webClient.getCookieManager().setCookiesEnabled(true);
            for (Map.Entry<String, String> cookieItem : pageRequest.getCookieMap().entrySet()) {
                webClient.getCookieManager().addCookie(new Cookie("", cookieItem.getKey(), cookieItem.getValue()));
            }
        }
        if (pageRequest.getHeaderMap() != null && !pageRequest.getHeaderMap().isEmpty()) {
            webRequest.setAdditionalHeaders(pageRequest.getHeaderMap());
        }
        if (pageRequest.getUserAgent() != null) {
            webRequest.setAdditionalHeader("User-Agent", pageRequest.getUserAgent());
        }
        if (pageRequest.getReferrer() != null) {
            webRequest.setAdditionalHeader("Referer", pageRequest.getReferrer());
        }

        webClient.getOptions().setTimeout(pageRequest.getTimeoutMillis());
        webClient.setJavaScriptTimeout(pageRequest.getTimeoutMillis());
        webClient.waitForBackgroundJavaScript(pageRequest.getTimeoutMillis());

        // 代理
        if (pageRequest.getProxy() != null) {
            InetSocketAddress address = (InetSocketAddress) pageRequest.getProxy().address();
            boolean isSocks = pageRequest.getProxy().type() == Proxy.Type.SOCKS;
            webClient.getOptions().setProxyConfig(new ProxyConfig(address.getHostName(), address.getPort(), isSocks));
        }

        // 发出请求
        if (pageRequest.isIfPost()) {
            webRequest.setHttpMethod(HttpMethod.POST);
        } else {
            webRequest.setHttpMethod(HttpMethod.GET);
        }
        HtmlPage page = webClient.getPage(webRequest);

        String pageAsXml = page.asXml();
        if (pageAsXml != null) {
            Document html = Jsoup.parse(pageAsXml);
            return html;
        }
    } catch (IOException e) {
        logger.error(e.getMessage(), e);
    } finally {
        if (webClient != null) {
            webClient.close();
        }
    }
    return null;
}