us.codecraft.webmagic.selector.Html Java Examples

@Override
public void process(Page page, HunterConfig model) {
    Html pageHtml = page.getHtml();
    String title = pageHtml.xpath(model.getTitleRegex()).get();
    String source = page.getRequest().getUrl();
    if (model.isSingle() || (!StringUtils.isEmpty(title) && (!"null".equals(title) && !model.getEntryUrls().contains(source)))) {
        page.putField("title", title);
        page.putField("source", source);
        this.put(page, pageHtml, "releaseDate", model.getReleaseDateRegex());
        this.put(page, pageHtml, "author", model.getAuthorRegex());
        this.put(page, pageHtml, "content", model.getContentRegex());
        this.put(page, pageHtml, "tags", model.getTagRegex());
        this.put(page, pageHtml, "description", model.getDescriptionRegex());
        this.put(page, pageHtml, "keywords", model.getKeywordsRegex());
    }
    if (!model.isSingle()) {
        if (StringUtils.isNotEmpty(model.getTargetLinksRegex())) {
            page.addTargetRequests(page.getHtml().links().regex(model.getTargetLinksRegex()).all());
        }
    }
}
 

/**
 * 获取所有图片标签的src连接
 *
 * @param html 原博客内容
 */
public static String formatHtml(String html) {
    if (StringUtils.isEmpty(html)) {
        return null;
    }
    String lazyloadFormat = "<img src=\"%s\" title=\"%s\" alt=\"%s\">";

    Html pageHtml = getHtml(html);
    List<Selectable> imgSelectables = pageHtml.$("img").nodes();
    for (Selectable imgSelectable : imgSelectables) {
        String oldImg = imgSelectable.get();
        String newImg = String.format(lazyloadFormat, getRealImgUrl(imgSelectable), imgSelectable.xpath("//img/@title").get(), imgSelectable.xpath("//img/@alt").get());
        html = html.replace(oldImg, newImg);
    }
    return html;
}
 

/**
 * 获取所有图片标签的src连接
 *
 * @param html 原博客内容
 */
public static Set<ImageLink> getAllImageLink(String html) {
    if (StringUtils.isEmpty(html)) {
        return null;
    }
    Set<ImageLink> imageLinks = new HashSet<>();
    ImageLink imageLink = null;

    Html pageHtml = getHtml(html);
    List<Selectable> imgSelectables = pageHtml.$("img").nodes();
    for (Selectable imgSelectable : imgSelectables) {
        String newImgSrc = getRealImgUrl(imgSelectable);
        imageLink = new ImageLink(newImgSrc);
        imageLinks.add(imageLink);
    }
    return imageLinks;
}
 

@Ignore
@Test
public void test() {
    ModelPageProcessor modelPageProcessor = ModelPageProcessor.create(Site.me(), OschinaBlog.class);
    Page page = new Page();
    page.setRequest(new Request("http://my.oschina.net/flashsword/blog"));
    page.setUrl(new PlainText("http://my.oschina.net/flashsword/blog"));
    page.setHtml(new Html(html));
    long time = System.currentTimeMillis();
    for (int i = 0; i < 1000; i++) {
        modelPageProcessor.process(page);
    }
    System.out.println(System.currentTimeMillis() - time);
    time = System.currentTimeMillis();
    for (int i = 0; i < 1000; i++) {
        modelPageProcessor.process(page);
    }
    System.out.println(System.currentTimeMillis() - time);
}
 

@Override
public void process(Page page) {
    List<String> relativeUrl = page.getHtml().xpath("//li[@class='item clearfix']/div/a/@href").all();
    page.addTargetRequests(relativeUrl);
    relativeUrl = page.getHtml().xpath("//div[@id='zh-question-related-questions']//a[@class='question_link']/@href").all();
    page.addTargetRequests(relativeUrl);
    List<String> answers =  page.getHtml().xpath("//div[@id='zh-question-answer-wrap']/div").all();
    boolean exist = false;
    for(String answer:answers){
        String vote = new Html(answer).xpath("//div[@class='zm-votebar']//span[@class='count']/text()").toString();
        if(Integer.valueOf(vote) >= voteNum){
            page.putField("vote",vote);
            page.putField("content",new Html(answer).xpath("//div[@class='zm-editable-content']"));
            page.putField("userid", new Html(answer).xpath("//a[@class='author-link']/@href"));
            exist = true;
        }
    }
    if(!exist){
        page.setSkip(true);
    }
}
 

private void put(Page page, Html pageHtml, String key, String regex) {
    if (StringUtils.isNotEmpty(regex)) {
        if (key.equals("tags")) {
            page.putField(key, pageHtml.xpath(regex).all());
            return;
        }
        page.putField(key, pageHtml.xpath(regex).get());
    }
}
 

public void process(Page page) {

        Html html = page.getHtml();
        List<String> questionList =  html.xpath("//table[@class='tgCustomerCommunityCenterColumn']//div[@class='content']//table[@class='dataGrid']//tr").all();

        if(questionList != null && questionList.size() > 1)
        {
            //i=0是列名称，所以i从1开始
            for( int i = 1 ; i < questionList.size(); i++)
            {
                System.out.println(questionList.get(i));
                Html tempHtml =  Html.create("<table>"+questionList.get(i)+"</table>");
                String comment = tempHtml.xpath("//td[@class='title']//a/text()").toString();
                System.out.println(comment);
                String answerNum =  tempHtml.xpath("//td[@class='num']/text()").toString();
                System.out.println(answerNum);
                String createTime = tempHtml.xpath("//td[3]/text()").toString();
                System.out.println(createTime);

				/* Document doc = Jsoup.parse(questionList.get(i));
				 Html hmt  = Html.create(questionList.get(i)) ;
			     String str = hmt.links().toString();
				  String   content =   doc.getElementsByTag("a").text();
				  String ss = doc.text();*/

            }
        }

    }
 

@Test
public void testNthNodesGet(){
	Html html = new Html("<a data-tip=\"p$t$xxx\" href=\"/xx/xx\">xx</a>");
	assertThat(html.xpath("//a[1]/@href").get()).isEqualTo("/xx/xx");
	Selectable selectable = html.xpath("//a[1]").nodes().get(0);
	assertThat(selectable.xpath("/a/@href").get()).isEqualTo("/xx/xx");
}
 

@Ignore("not work in jsoup 1.8.x")
@Test
public void testDisableJsoupHtmlEntityEscape() throws Exception {
	Html.DISABLE_HTML_ENTITY_ESCAPE = true;
	Html html = new Html("aaaaaaa&b");
	assertThat(html.regex("(aaaaaaa&b)").toString()).isEqualTo("aaaaaaa&b");
}
 

@Test
  public void testRegexSelector() {
      Html selectable = new Html("aaaaaaab");
assertThat(selectable.regex("(a+b)").replace("aa(a)", "$1bb").toString()).isEqualTo("abbabbab");
  }
 

@Test
public void testEnableJsoupHtmlEntityEscape() throws Exception {
	Html html = new Html("aaaaaaa&b");
	assertThat(html.regex("(aaaaaaa&b)").toString()).isEqualTo("aaaaaaa&b");
}
 

@Test
public void testAHrefExtract(){
	Html html = new Html("<a data-tip=\"p$t$xxx\" href=\"/xx/xx\">xx</a>");
	assertThat(html.links().all()).contains("/xx/xx");
}
 

@Test
public void testDownloader() {
    HttpClientDownloader httpClientDownloader = new HttpClientDownloader();
    Html html = httpClientDownloader.download("https://www.baidu.com/");
    assertTrue(!html.getFirstSourceText().isEmpty());
}
 

private static Html getHtml(String html) {
    Page page = new Page();
    page.setRequest(new Request(""));
    page.setRawText(html);
    return page.getHtml();
}
 

/**
 * A simple method to download a url.
 *
 * @param url url
 * @param charset charset
 * @return html
 */
public Html download(String url, String charset) {
    Page page = download(new Request(url), Site.me().setCharset(charset).toTask());
    return (Html) page.getHtml();
}
 

/**
 * A simple method to download a url.
 *
 * @param url url
 * @return html
 */
public Html download(String url) {
    return download(url, null);
}
 

/**
 * @param html html
 * @deprecated since 0.4.0
 * The html is parse just when first time of calling {@link #getHtml()}, so use {@link #setRawText(String)} instead.
 */
public void setHtml(Html html) {
    this.html = html;
}