Java Code Examples for com.gargoylesoftware.htmlunit.WebClient#waitForBackgroundJavaScript()

private void getCurrentPosition(final boolean geolocationEnabled) throws Exception {
    final Map<String, Class<? extends Servlet>> servlets = new HashMap<>();
    servlets.put("/test", GetCurrentPositionTestServlet.class);
    servlets.put("/browserLocation", BrowserLocationServlet.class);
    startWebServer("./", new String[0], servlets);

    Geolocation.setProviderUrl(URL_FIRST + "browserLocation");
    final WebClient client = getWebClient();
    if (geolocationEnabled) {
        client.getOptions().setGeolocationEnabled(true);
    }
    final List<String> collectedAlerts = new ArrayList<>();
    client.setAlertHandler(new CollectingAlertHandler(collectedAlerts));
    client.getPage(URL_FIRST + "test");
    client.waitForBackgroundJavaScript(2000);

    assertEquals(getExpectedAlerts(), collectedAlerts);
}
 

/**
 * Returns a list of channel IDs that are displayed on the channels.html page for the bounce proxy.
 * 
 * @param webClient
 * @param url
 * @return
 * @throws Exception
 */
public static List<String> getChannelIdsOnChannelsHtml(WebClient webClient, String url) throws Exception {

    HtmlPage page = webClient.getPage(url);
    webClient.waitForBackgroundJavaScript(2000);

    DomElement channelsTable = page.getElementById("channels");

    List<String> channelIds = new LinkedList<String>();
    for (DomElement channelsTableRows : channelsTable.getChildElements()) {
        if (channelsTableRows.getTagName().equals("tbody")) {

            for (DomElement channelRows : channelsTableRows.getChildElements()) {

                String channelId = channelRows.getChildNodes().get(0).getTextContent();
                if (isProperChannelId(channelId)) {
                    channelIds.add(channelId);
                }
            }
        }
    }
    return channelIds;
}
 

/**
 * Test for a strange error we found: An ajax running
 * in parallel shares the additional headers with a form
 * submit.
 *
 * @throws Exception if an error occurs
 */
@Test
public void ajaxInfluencesSubmitHeaders() throws Exception {
    final Map<String, Class<? extends Servlet>> servlets = new HashMap<>();
    servlets.put("/content.html", ContentServlet.class);
    servlets.put("/ajax_headers.html", AjaxHeaderServlet.class);
    servlets.put("/form_headers.html", FormHeaderServlet.class);
    startWebServer("./", null, servlets);

    collectedHeaders_.clear();
    XMLHttpRequest3Test.STATE_ = 0;
    final WebClient client = getWebClient();

    final List<String> collectedAlerts = Collections.synchronizedList(new ArrayList<String>());
    client.setAlertHandler(new CollectingAlertHandler(collectedAlerts));

    final HtmlPage page = client.getPage(URL_FIRST + "content.html");
    final DomElement elem = page.getElementById("doIt");
    while (STATE_ < 1) {
        Thread.sleep(42);
    }
    ((HtmlSubmitInput) elem).click();

    client.waitForBackgroundJavaScript(DEFAULT_WAIT_TIME);
    assertEquals(collectedHeaders_.toString(), 2, collectedHeaders_.size());

    String headers = collectedHeaders_.get(0);
    if (!headers.startsWith("Form: ")) {
        headers = collectedHeaders_.get(1);
    }
    assertTrue(headers, headers.startsWith("Form: "));
    assertFalse(headers, headers.contains("Html-Unit=is great,;"));

    headers = collectedHeaders_.get(0);
    if (!headers.startsWith("Ajax: ")) {
        headers = collectedHeaders_.get(1);
    }
    assertTrue(headers, headers.startsWith("Ajax: "));
    assertTrue(headers, headers.contains("Html-Unit=is great,;"));
}
 

public static void main(String[] args) throws FailingHttpStatusCodeException, MalformedURLException, IOException {
    
    // 屏蔽HtmlUnit等系统 log
    LogFactory.getFactory().setAttribute("org.apache.commons.logging.Log","org.apache.commons.logging.impl.NoOpLog");
    java.util.logging.Logger.getLogger("com.gargoylesoftware").setLevel(Level.OFF);
    java.util.logging.Logger.getLogger("org.apache.http.client").setLevel(Level.OFF);
    
    String url = "https://www.newsmth.net/nForum/#!section/Estate";
    System.out.println("Loading page now-----------------------------------------------: "+url);
    
    /* HtmlUnit 模拟浏览器 */
    WebClient webClient = new WebClient(BrowserVersion.CHROME);
    webClient.getOptions().setJavaScriptEnabled(true);              // 启用JS解释器，默认为true  
    webClient.getOptions().setCssEnabled(false);                    // 禁用css支持  
    webClient.getOptions().setThrowExceptionOnScriptError(false);   // js运行错误时，是否抛出异常
    webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
    webClient.getOptions().setTimeout(10 * 1000);                   // 设置连接超时时间
    HtmlPage page = webClient.getPage(url);
    webClient.waitForBackgroundJavaScript(30 * 1000);               // 等待js后台执行30秒

    String pageAsXml = page.asXml();
    
    /* Jsoup解析处理 */
    // Document doc = Jsoup.parse(pageAsXml, "https://bluetata.com/");
    Document doc = Jsoup.parse(pageAsXml);  
    //Elements pngs = doc.select("img[src$=.png]");                   // 获取所有图片元素集
    
    Elements eles = doc.select("td.title_1");
    // 其他操作
    System.out.println(eles.toString());
}
 

@Override
public Document load(PageRequest pageRequest) {
    if (!UrlUtil.isUrl(pageRequest.getUrl())) {
        return null;
    }

    WebClient webClient = new WebClient();
    try {
        WebRequest webRequest = new WebRequest(new URL(pageRequest.getUrl()));

        // 请求设置
        webClient.getOptions().setUseInsecureSSL(true);
        webClient.getOptions().setJavaScriptEnabled(true);
        webClient.getOptions().setCssEnabled(false);
        webClient.getOptions().setThrowExceptionOnScriptError(false);
        webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
        webClient.getOptions().setDoNotTrackEnabled(false);
        webClient.getOptions().setUseInsecureSSL(!pageRequest.isValidateTLSCertificates());

        if (pageRequest.getParamMap() != null && !pageRequest.getParamMap().isEmpty()) {
            for (Map.Entry<String, String> paramItem : pageRequest.getParamMap().entrySet()) {
                webRequest.getRequestParameters().add(new NameValuePair(paramItem.getKey(), paramItem.getValue()));
            }
        }
        if (pageRequest.getCookieMap() != null && !pageRequest.getCookieMap().isEmpty()) {
            webClient.getCookieManager().setCookiesEnabled(true);
            for (Map.Entry<String, String> cookieItem : pageRequest.getCookieMap().entrySet()) {
                webClient.getCookieManager().addCookie(new Cookie("", cookieItem.getKey(), cookieItem.getValue()));
            }
        }
        if (pageRequest.getHeaderMap() != null && !pageRequest.getHeaderMap().isEmpty()) {
            webRequest.setAdditionalHeaders(pageRequest.getHeaderMap());
        }
        if (pageRequest.getUserAgent() != null) {
            webRequest.setAdditionalHeader("User-Agent", pageRequest.getUserAgent());
        }
        if (pageRequest.getReferrer() != null) {
            webRequest.setAdditionalHeader("Referer", pageRequest.getReferrer());
        }

        webClient.getOptions().setTimeout(pageRequest.getTimeoutMillis());
        webClient.setJavaScriptTimeout(pageRequest.getTimeoutMillis());
        webClient.waitForBackgroundJavaScript(pageRequest.getTimeoutMillis());

        // 代理
        if (pageRequest.getProxy() != null) {
            InetSocketAddress address = (InetSocketAddress) pageRequest.getProxy().address();
            boolean isSocks = pageRequest.getProxy().type() == Proxy.Type.SOCKS;
            webClient.getOptions().setProxyConfig(new ProxyConfig(address.getHostName(), address.getPort(), isSocks));
        }

        // 发出请求
        if (pageRequest.isIfPost()) {
            webRequest.setHttpMethod(HttpMethod.POST);
        } else {
            webRequest.setHttpMethod(HttpMethod.GET);
        }
        HtmlPage page = webClient.getPage(webRequest);

        String pageAsXml = page.asXml();
        if (pageAsXml != null) {
            Document html = Jsoup.parse(pageAsXml);
            return html;
        }
    } catch (IOException e) {
        logger.error(e.getMessage(), e);
    } finally {
        if (webClient != null) {
            webClient.close();
        }
    }
    return null;
}