Java Code Examples for org.jsoup.select.Elements#addAll()

/**
 * 获取内容列表
 */
private List<String> getResultList(String ruleStr) {
    if (isEmpty(ruleStr)) {
        return null;
    }
    Elements elements = new Elements();
    elements.add(element);
    String[] rules = ruleStr.split("@");
    for (int i = 0; i < rules.length - 1; i++) {
        Elements es = new Elements();
        for (Element elt : elements) {
            es.addAll(getElementsSingle(elt, rules[i]));
        }
        elements.clear();
        elements = es;
    }
    if (elements.isEmpty()) {
        return null;
    }
    return getResultLast(elements, rules[rules.length - 1]);
}
 

/**
 * 获取内容列表
 */
private List<String> getResultList(String ruleStr) {
    if (isEmpty(ruleStr)) {
        return null;
    }
    Elements elements = new Elements();
    elements.add(element);
    String[] rules = ruleStr.split("@");
    for (int i = 0; i < rules.length - 1; i++) {
        Elements es = new Elements();
        for (Element elt : elements) {
            es.addAll(getElementsSingle(elt, rules[i]));
        }
        elements.clear();
        elements = es;
    }
    if (elements.isEmpty()) {
        return null;
    }
    return getResultLast(elements, rules[rules.length - 1]);
}
 

private List<String> parseStringList(Element element, String rule) {
    final List<String> textS = new ArrayList<>();
    Elements elements = new Elements();
    elements.add(element);
    String[] ruleS = rule.split("@");
    for (int i = 0, length = ruleS.length - 1; i < length; i++) {
        Elements es = new Elements();
        for (Element elt : elements) {
            es.addAll(parseList(elt, ruleS[i]));
        }
        elements.clear();
        elements.addAll(es);
    }
    if (!elements.isEmpty()) {
        return parseLastResult(elements, ruleS[ruleS.length - 1]);
    }
    return textS;
}
 

/**
 * 函数具体逻辑
 *
 * @param scope 上下文
 * @return 计算好的节点
 */
@Override
public XValue call(Scope scope) {
    Elements context = new Elements();
    for (Element el:scope.context()){
        context.addAll(el.children());
        String  txt = el.ownText();
        if (StringUtils.isNotBlank(txt)){
            Element et = new Element("");
            et.appendText(txt);
            context.add(et);
        }
    }
    return XValue.create(context);
}
 

public static Elements selectElements(Element element, String rule) {
    String[] ors = rule.split("\\|\\|");
    Elements res = new Elements();
    for (int i = 0; i < ors.length; i++) {
        try {
            res.addAll(selectElementsWithoutOr(element, ors[i]));
        } catch (Exception e1) {
            e1.printStackTrace();
        }
    }
    return res;
}
 

/**
 *
 * @param context
 * @return res
 */
@Override
public XValue apply(Elements context) {
    List<Element> total = new LinkedList<>();
    for (Element el : context){
        if (el.nextElementSibling()!=null){
            total.add(el.nextElementSibling());
        }
    }
    Elements newContext = new Elements();
    newContext.addAll(total);
    return XValue.create(newContext);
}
 

public Elements getElements() {
    Elements elements = new Elements();
    for (XElements xElements : elementsList) {
        elements.addAll(xElements.getElements());
    }
    return elements;
}
 

/**
 * @param context
 * @return res
 */
@Override
public XValue apply(Elements context) {
    List<Element> total = new LinkedList<>();
    for (Element el : context){
        Elements ps = CommonUtil.precedingSibling(el);
        if (ps == null){
            continue;
        }
        total.addAll(ps);
    }
    Elements newContext = new Elements();
    newContext.addAll(total);
    return XValue.create(newContext);
}
 

@Override
public XValue apply(Elements context) {
    Elements childs = new Elements();
    for (Element el:context){
        childs.addAll(el.children());
    }
    return XValue.create(childs);
}
 

/**
 *
 * @param context
 * @return res
 */
@Override
public XValue apply(Elements context) {
    List<Element> total = new LinkedList<>();
    for (Element el : context){
        Elements fs = CommonUtil.followingSibling(el);
        if (fs == null){
            continue;
        }
        total.addAll(fs);
    }
    Elements newContext = new Elements();
    newContext.addAll(total);
    return XValue.create(newContext);
}
 

/**
 * @param context
 * @return res
 */
@Override
public XValue apply(Elements context) {
    List<Element> total = new LinkedList<>();
    for (Element el : context){
        if (el.previousElementSibling()!=null){
            total.add(el);
        }
    }
    Elements newContext = new Elements();
    newContext.addAll(total);
    return XValue.create(newContext);
}
 

@Override
public XValue apply(Elements context) {
    Set<Element> total = new HashSet<>();
    Elements descendant = new Elements();
    for (Element el:context){
        Elements tmp = el.getAllElements();
        total.addAll(tmp);
    }
    descendant.addAll(total);
    return XValue.create(descendant);
}
 

@Override
public XValue apply(Elements context) {
    Set<Element> total = new HashSet<>();
    Elements descendant = new Elements();
    for (Element el:context){
        Elements tmp = el.getAllElements();
        //exclude self
        tmp.remove(el);
        total.addAll(tmp);
    }
    descendant.addAll(total);
    return XValue.create(descendant);
}
 

@Override
public XValue visitAbbreviatedStep(XpathParser.AbbreviatedStepContext ctx) {
    if ("..".equals(ctx.getText())){
        Set<Element> total = new HashSet<>();
        Elements newContext = new Elements();
        for (Element e:currentScope().context()){
            total.add(e.parent());
        }
        newContext.addAll(total);
        return XValue.create(newContext);
    }else {
        return XValue.create(currentScope().context());
    }
}
 

/**
 * 
 * @param el
 * @return all the parents and the siblings of the element
 */
private Elements getSiblingsAndParents(Element el) {
    Elements siblingsAndParents = new Elements();
    siblingsAndParents.addAll(el.siblingElements());
    siblingsAndParents.addAll(el.parents());
    return siblingsAndParents;
}
 

private Elements parseContent( String content )
{
    Elements alerts = new Elements();
    Document doc = Jsoup.parse( content );
    Elements alertDivs = doc.select( "div[class=alert-content]" );
    for ( Element element : alertDivs )
    {
        if ( !element.text().toLowerCase().contains( "elevator" ) )
        {
            alerts.addAll( element.select( "p[class=veh-replace]" ) );
        }
    }
    return alerts;
}
 

private static SimpleListBean parseFavorites(Document doc) {
    if (doc == null) {
        return null;
    }

    SimpleListBean list = new SimpleListBean();

    int last_page = 1;
    //if this is the last page, page number is in <strong>
    Elements pagesES = doc.select("div.pages a");
    pagesES.addAll(doc.select("div.pages strong"));
    if (pagesES.size() > 0) {
        for (Node n : pagesES) {
            int tmp = Utils.getIntFromString(((Element) n).text());
            if (tmp > last_page) {
                last_page = tmp;
            }
        }
    }
    list.setMaxPage(last_page);

    Elements trES = doc.select("table.datatable tbody tr");
    for (int i = 0; i < trES.size(); ++i) {
        Element trE = trES.get(i);
        SimpleListItemBean item = new SimpleListItemBean();

        Elements subjectES = trE.select("th");
        if (subjectES.size() == 0) {
            continue;
        }
        item.setTitle(subjectES.first().text());

        Elements subjectAES = subjectES.first().select("a");
        if (subjectAES.size() == 0) {
            continue;
        }
        String href = subjectAES.first().attr("href");
        item.setTid(Utils.getMiddleString(href, "tid=", "&"));

        Elements timeES = trE.select("td.lastpost");
        if (timeES.size() > 0) {
            item.setTime(timeES.first().text().trim());
        }

        Elements forumES = trE.select("td.forum");
        if (forumES.size() > 0) {
            item.setForum(forumES.first().text().trim());
        }

        list.add(item);
    }

    return list;
}
 

private static SimpleListBean parseSearchFullText(Document doc) {
        if (doc == null) {
            return null;
        }

        SimpleListBean list = new SimpleListBean();
        int last_page = 1;

        //if this is the last page, page number is in <strong>
        Elements pagesES = doc.select("div.pages_btns div.pages a");
        pagesES.addAll(doc.select("div.pages_btns div.pages strong"));
        String searchIdUrl;
        if (pagesES.size() > 0) {
            searchIdUrl = pagesES.first().attr("href");
            list.setSearchId(Utils.getMiddleString(searchIdUrl, "searchid=", "&"));
            for (Node n : pagesES) {
                int tmp = Utils.getIntFromString(((Element) n).text());
                if (tmp > last_page) {
                    last_page = tmp;
                }
            }
        }
        list.setMaxPage(last_page);

        Elements tbodyES = doc.select("table.datatable tr");
        for (int i = 0; i < tbodyES.size(); ++i) {
            Element trowE = tbodyES.get(i);
            SimpleListItemBean item = new SimpleListItemBean();

            Elements subjectES = trowE.select("div.sp_title a");
            if (subjectES.size() == 0) {
                continue;
            }
            item.setTitle(subjectES.first().text());
            //gotopost.php?pid=12345
            String postUrl = Utils.nullToText(subjectES.first().attr("href"));
            item.setPid(Utils.getMiddleString(postUrl, "pid=", "&"));
            if (TextUtils.isEmpty(item.getPid())) {
                continue;
            }

            Elements contentES = trowE.select("div.sp_content");
            if (contentES.size() > 0) {
                item.setInfo(contentES.text());
            }

//            <div class="sp_theard">
//            <span class="sp_w200">版块: <a href="forumdisplay.php?fid=2">Discovery</a></span>
//            <span>作者: <a href="space.php?uid=189027">tsonglin</a></span>
//            <span>查看: 1988</span>
//            <span>回复: 56</span>
//            <span class="sp_w200">最后发表: 2015-4-4 21:58</span>
//            </div>
            Elements postInfoES = trowE.select("div.sp_theard span");
            if (postInfoES.size() != 5) {
                continue;
            }
            Elements authorES = postInfoES.get(1).select("a");
            if (authorES.size() > 0) {
                item.setAuthor(authorES.first().text());
                String spaceUrl = authorES.first().attr("href");
                if (!TextUtils.isEmpty(spaceUrl)) {
                    String uid = Utils.getMiddleString(spaceUrl, "uid=", "&");
                    item.setAvatarUrl(HiUtils.getAvatarUrlByUid(uid));
                }
            }

            item.setTime(Utils.getMiddleString(postInfoES.get(4).text(), ":", "&"));

            Elements forumES = postInfoES.get(0).select("a");
            if (forumES.size() > 0)
                item.setForum(forumES.first().text());

            list.add(item);
        }

        return list;
    }
 

private static SimpleListBean parseSearch(Document doc) {
    if (doc == null) {
        return null;
    }

    SimpleListBean list = new SimpleListBean();
    int last_page = 1;

    //if this is the last page, page number is in <strong>
    Elements pagesES = doc.select("div.pages_btns div.pages a");
    pagesES.addAll(doc.select("div.pages_btns div.pages strong"));
    String searchIdUrl;
    if (pagesES.size() > 0) {
        searchIdUrl = pagesES.first().attr("href");
        list.setSearchId(Utils.getMiddleString(searchIdUrl, "searchid=", "&"));
        for (Node n : pagesES) {
            int tmp = Utils.getIntFromString(((Element) n).text());
            if (tmp > last_page) {
                last_page = tmp;
            }
        }
    }
    list.setMaxPage(last_page);

    Elements tbodyES = doc.select("tbody");
    for (int i = 0; i < tbodyES.size(); ++i) {
        Element tbodyE = tbodyES.get(i);
        SimpleListItemBean item = new SimpleListItemBean();

        Elements subjectES = tbodyE.select("tr th.subject a");
        if (subjectES.size() == 0) {
            continue;
        }

        Element titleLink = subjectES.first();
        String href = titleLink.attr("href");
        item.setTid(Utils.getMiddleString(href, "tid=", "&"));
        item.setTitle(titleLink.text());

        Elements authorAES = tbodyE.select("tr td.author cite a");
        if (authorAES.size() == 0) {
            continue;
        }
        item.setAuthor(authorAES.first().text());

        String spaceUrl = authorAES.first().attr("href");
        if (!TextUtils.isEmpty(spaceUrl)) {
            String uid = Utils.getMiddleString(spaceUrl, "uid=", "&");
            item.setAvatarUrl(HiUtils.getAvatarUrlByUid(uid));
        }

        Elements timeES = tbodyE.select("tr td.author em");
        if (timeES.size() > 0) {
            item.setTime(timeES.first().text());
        }

        Elements forumES = tbodyE.select("tr td.forum");
        if (forumES.size() > 0) {
            item.setForum(forumES.first().text());
        }

        list.add(item);
    }

    return list;
}
 

private static SimpleListBean parseMyPost(Document doc) {
    if (doc == null) {
        return null;
    }

    Elements tableES = doc.select("table.datatable");
    if (tableES.size() == 0) {
        return null;
    }

    SimpleListBean list = new SimpleListBean();

    int last_page = 1;
    //if this is the last page, page number is in <strong>
    Elements pagesES = doc.select("div.pages_btns div.pages a");
    pagesES.addAll(doc.select("div.pages_btns div.pages strong"));
    if (pagesES.size() > 0) {
        for (Node n : pagesES) {
            int tmp = Utils.getIntFromString(((Element) n).text());
            if (tmp > last_page) {
                last_page = tmp;
            }
        }
    }
    list.setMaxPage(last_page);

    Elements trES = tableES.first().select("tr");

    SimpleListItemBean item = null;
    //first tr is title, skip
    for (int i = 1; i < trES.size(); ++i) {
        Element trE = trES.get(i);

        // odd have title, even have reply text;
        item = new SimpleListItemBean();

        // thread
        Elements thES = trE.select("th");
        if (thES.size() == 0) {
            continue;
        }
        Elements linkES = thES.first().select("a");
        if (linkES.size() != 1) {
            continue;
        }
        String tid = linkES.first().attr("href");
        if (!tid.contains("viewthread.php?tid=")) {
            continue;
        }
        tid = Utils.getMiddleString(tid, "viewthread.php?tid=", "&");
        String title = linkES.first().text();

        // time
        Elements lastpostES = trE.select("td.lastpost");
        if (lastpostES.size() == 0) {
            continue;
        }
        String time = lastpostES.first().text();

        item.setTid(tid);
        item.setTitle(title);
        item.setTime(time);

        Elements forumES = trE.select("td.forum");
        if (forumES.size() > 0) {
            item.setForum(forumES.first().text());
        }

        list.add(item);
    }
    return list;
}