Java Code Examples for org.jsoup.Connection#data()

public static String post(String url, Map<String, String> headers,
		Map<String, String> params) {
	try {
		Connection connect = Jsoup.connect(url);
		connect.ignoreContentType(true);
		connect.ignoreHttpErrors(true);
		if (params != null) {
			connect.data(params);
		}
		if (headers != null) {
			connect.headers(headers);
		}
		connect.method(Method.POST);
		return connect.execute().body();
	} catch (Throwable e) {
	}
	return null;
}
 

@Override
protected List<Winner> doInBackground(Void... params) {
    Log.d(TAG, "Fetching giveaways for page " + page);

    try {
        // Fetch the Giveaway page

        Connection jsoup = Jsoup.connect("https://www.steamgifts.com/giveaway/" + path + "/winners/search")
                .userAgent(Constants.JSOUP_USER_AGENT)
                .timeout(Constants.JSOUP_TIMEOUT);
        jsoup.data("page", Integer.toString(page));

        if (SteamGiftsUserData.getCurrent(fragment.getContext()).isLoggedIn())
            jsoup.cookie("PHPSESSID", SteamGiftsUserData.getCurrent(fragment.getContext()).getSessionId());
        Document document = jsoup.get();

        SteamGiftsUserData.extract(fragment.getContext(), document);

        return loadAll(document);
    } catch (IOException e) {
        Log.e(TAG, "Error fetching URL", e);
        return null;
    }
}
 

/**
 * Method which publishes a link to shaarli
 * Assume being logged in
 * TODO: use the prefetch function
 */
@Override
public void pushLink(Link link) throws IOException {
    String encodedShareUrl = URLEncoder.encode(link.getUrl(), "UTF-8");
    retrievePostLinkToken(encodedShareUrl);

    if (NetworkUtils.isUrl(link.getUrl())) { // In case the url isn't really one, just post the one chosen by the server.
        this.mSharedUrl = link.getUrl();
    }

    final String postUrl = this.mShaarliUrl + "?post=" + encodedShareUrl;

    Connection postPageConn = this.newConnection(postUrl, Connection.Method.POST)
            .data("save_edit", "Save")
            .data("token", this.mToken)
            .data("lf_tags", link.getTags())
            .data("lf_linkdate", this.mDatePostLink)
            .data("lf_url", this.mSharedUrl)
            .data("lf_title", link.getTitle())
            .data("lf_description", link.getDescription());
    if (link.isPrivate()) postPageConn.data("lf_private", "on");
    if (link.isTweet()) postPageConn.data("tweet", "on");
    if (link.isToot()) postPageConn.data("toot", "on");
    postPageConn.execute(); // Then we post
}
 

/**
 * Test fetching a form, and submitting it with a file attached.
 */
@Test
public void postHtmlFile() throws IOException {
    Document index = Jsoup.connect("http://direct.infohound.net/tidy/").get();
    FormElement form = index.select("[name=tidy]").forms().get(0);
    Connection post = form.submit();

    File uploadFile = ParseTest.getFile("/htmltests/google-ipod.html");
    FileInputStream stream = new FileInputStream(uploadFile);
    
    Connection.KeyVal fileData = post.data("_file");
    fileData.value("check.html");
    fileData.inputStream(stream);

    Connection.Response res;
    try {
        res = post.execute();
    } finally {
        stream.close();
    }

    Document out = res.parse();
    assertTrue(out.text().contains("HTML Tidy Complete"));
}
 

@Test
public void followsRedirectToHttps() throws IOException {
    Connection con = Jsoup.connect("http://direct.infohound.net/tools/302-secure.pl"); // https://www.google.com
    con.data("id", "5");
    Document doc = con.get();
    assertTrue(doc.title().contains("Google"));
}
 

public void postJspToHtml(String postUrl, String filePath,String fileName) throws Exception{
	HttpServletRequest request=Struts2Utils.getRequest();
	//${pageContext.request.scheme}://${pageContext.request.serverName }:${pageContext.request.serverPort} pageContext.request.contextPath
	String reqTarget = request.getScheme()+"://"+request.getServerName()+(request.getServerPort()==80?"":":"+request.getServerPort())+request.getContextPath();
	reqTarget =reqTarget+"/toHtml";
	//?url="+postUrl+"&filePath="+filePath+"&fileName="+fileName;
	Map<String, String> map=new HashMap<String, String>();
	map.put("url", postUrl);
	map.put("filePath", filePath);
	map.put("fileName", fileName);
	Connection connection = Jsoup.connect(reqTarget);
	connection.userAgent("Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.64 Safari/537.31");
	connection.data(map);
	Document doc=connection.timeout(8000).get();
}
 

private Response sendRequest(Method method, String apiPath, boolean absoluteApiPath, String... keyval) throws IOException {
  String url = absoluteApiPath ? apiPath : SERVER_HOSTNAME + apiPath;
  Connection conn = Jsoup.connect(url).maxBodySize(100 * 1024 * 1024).timeout(10000).method(method).ignoreContentType(true).ignoreHttpErrors(true);
  logger.finest("Sending " + method + " request at " + url);
  if (skypeToken != null) {
    conn.header("X-Skypetoken", skypeToken);
  } else {
    logger.fine("No token sent for the request at: " + url);
  }
  conn.data(keyval);
  return conn.execute();
}
 

@Override
protected List<IEndlessAdaptable> doInBackground(Void... params) {
    try {
        // Fetch the Giveaway page

        Connection jsoup = Jsoup.connect("https://www.steamgifts.com/" + pathSegment + "/search")
                .userAgent(Constants.JSOUP_USER_AGENT)
                .timeout(Constants.JSOUP_TIMEOUT);
        jsoup.data("page", Integer.toString(page));

        if (searchQuery != null)
            jsoup.data("q", searchQuery);

        jsoup.cookie("PHPSESSID", SteamGiftsUserData.getCurrent(context).getSessionId());

        Document document = jsoup.get();

        SteamGiftsUserData.extract(context, document);

        // Fetch the xsrf token
        Element xsrfToken = document.select("input[name=xsrf_token]").first();
        if (xsrfToken != null)
            foundXsrfToken = xsrfToken.attr("value");

        // Do away with pinned giveaways.
        document.select(".pinned-giveaways__outer-wrap").html("");

        // Parse all rows of giveaways
        return loadAll(document);
    } catch (Exception e) {
        Log.e(TAG, "Error fetching URL", e);
        return null;
    }
}
 

@Test(expected=IllegalArgumentException.class) public void throwsOnOdddData() {
    Connection con = HttpConnection.connect("http://example.com/");
    con.data("Name", "val", "what");
}
 

@Override
protected List<GiveawayGroup> doInBackground(Void... params) {
    Log.d(TAG, "Fetching giveaways for page " + page);

    try {
        // Fetch the Giveaway page

        Connection jsoup = Jsoup.connect("https://www.steamgifts.com/giveaway/" + path + "/groups/search")
                .userAgent(Constants.JSOUP_USER_AGENT)
                .timeout(Constants.JSOUP_TIMEOUT);
        jsoup.data("page", Integer.toString(page));

        if (SteamGiftsUserData.getCurrent(fragment.getContext()).isLoggedIn())
            jsoup.cookie("PHPSESSID", SteamGiftsUserData.getCurrent(fragment.getContext()).getSessionId());
        Document document = jsoup.get();

        SteamGiftsUserData.extract(fragment.getContext(), document);

        // Parse all rows of groups
        Elements groups = document.select(".table__row-inner-wrap");
        Log.d(TAG, "Found inner " + groups.size() + " elements");

        List<GiveawayGroup> groupList = new ArrayList<>();
        for (Element element : groups) {
            Element link = element.select(".table__column__heading").first();

            // Basic information
            String title = link.text();
            String id = link.attr("href").substring(7, 12);

            String avatar = null;
            Element avatarNode = element.select(".global__image-inner-wrap").first();
            if (avatarNode != null)
                avatar = Utils.extractAvatar(avatarNode.attr("style"));

            GiveawayGroup group = new GiveawayGroup(id, title, avatar);
            groupList.add(group);
        }

        return groupList;
    } catch (IOException e) {
        Log.e(TAG, "Error fetching URL", e);
        return null;
    }
}
 

@Override
protected void addExtraParameters(Connection connection) {
    connection.data("game_id", String.valueOf(internalGameId));
}
 

@Override
protected List<Comment> doInBackground(Void... params) {
    Log.d(TAG, "Fetching giveaways for user " + steamID64 + " (" + rating + ") on page " + page);

    try {
        // Fetch the Giveaway page
        Connection connection = Jsoup.connect("https://www.steamtrades.com/user/" + steamID64 + "/search")
                .userAgent(Constants.JSOUP_USER_AGENT)
                .timeout(Constants.JSOUP_TIMEOUT);
        connection.data("page", Integer.toString(page));
        connection.data("rating", rating);

        /* FIXME broken with the split of steamtrades & steamgifts
        if (SteamGiftsUserData.getCurrent(fragment.getContext()).isLoggedIn()) {
            connection.cookie("PHPSESSID", SteamGiftsUserData.getCurrent(fragment.getContext()).getSessionId());
            connection.followRedirects(false);
        }
        */

        Connection.Response response = connection.execute();
        Document document = response.parse();

        if (response.statusCode() == 200) {
            // FIXME SteamGiftsUserData.extract(fragment.getContext(), document);

            //if (!user.isLoaded())
                //foundXsrfToken = Utils.loadUserProfile(user, document);

            user.setPositiveFeedback(Utils.parseInt(document.select(".increment_positive_review_count").text()));
            user.setNegativeFeedback(Utils.parseInt(document.select(".increment_negative_review_count").text()));


            Element rootCommentNode = document.select(".reviews").first();
            if (rootCommentNode != null) {
                // Parse all rows of giveaways
                ICommentHolder holder = new ICommentHolder() {
                    private List<Comment> list = new ArrayList<>();

                    @Override
                    public List<Comment> getComments() {
                        return list;
                    }

                    @Override
                    public void addComment(Comment comment) {
                        list.add(comment);
                    }
                };
                Utils.loadComments(rootCommentNode, holder, 0, false, true, Comment.Type.TRADE_FEEDBACK);
                return holder.getComments();
            } else
                return new ArrayList<>();
        } else {
            Log.w(TAG, "Got status code " + response.statusCode());
            return null;
        }
    } catch (Exception e) {
        Log.e(TAG, "Error fetching URL", e);
        return null;
    }
}
 

private void addFilterParameter(Connection jsoup, String parameterName, int value) {
    if (value >= 0)
        jsoup.data(parameterName, String.valueOf(value));
}
 

@Override
protected List<Discussion> doInBackground(Void... params) {
    try {
        // Fetch the Giveaway page
        String segment = "";
        if (type != DiscussionListFragment.Type.ALL)
            segment = type.name().replace("_", "-").toLowerCase(Locale.ENGLISH) + "/";
        String url = "https://www.steamgifts.com/discussions/" + segment + "search";

        Log.d(TAG, "Fetching discussions for page " + page + " and URL " + url);

        Connection jsoup = Jsoup.connect(url)
                .userAgent(Constants.JSOUP_USER_AGENT)
                .timeout(Constants.JSOUP_TIMEOUT);
        jsoup.data("page", Integer.toString(page));

        if (searchQuery != null)
            jsoup.data("q", searchQuery);

        // We do not want to follow redirects here, because SteamGifts redirects to the main (giveaways) page if we're not logged in.
        // For all other pages however, if we're not logged in, we're redirected once as well?
        if (type == DiscussionListFragment.Type.CREATED)
            jsoup.followRedirects(false);

        if (SteamGiftsUserData.getCurrent(fragment.getContext()).isLoggedIn())
            jsoup.cookie("PHPSESSID", SteamGiftsUserData.getCurrent(fragment.getContext()).getSessionId());
        Document document = jsoup.get();

        SteamGiftsUserData.extract(fragment.getContext(), document);

        // Parse all rows of discussions
        Elements discussions = document.select(".table__row-inner-wrap");
        Log.d(TAG, "Found inner " + discussions.size() + " elements");

        List<Discussion> discussionList = new ArrayList<>();
        for (Element element : discussions) {
            Element link = element.select("h3 a").first();

            // Basic information
            Uri uri = Uri.parse(link.attr("href"));
            String discussionId = uri.getPathSegments().get(1);
            String discussionName = uri.getPathSegments().get(2);

            Discussion discussion = new Discussion(discussionId);
            discussion.setTitle(link.text());
            discussion.setName(discussionName);

            Element p = element.select(".table__column--width-fill p").first();
            discussion.setCreatedTime(Integer.valueOf(p.select("span").first().attr("data-timestamp")));
            discussion.setCreator(p.select("a").last().text());

            // The creator's avatar
            Element avatarNode = element.select(".table_image_avatar").first();
            if (avatarNode != null)
                discussion.setCreatorAvatar(Utils.extractAvatar(avatarNode.attr("style")));

            discussion.setLocked(element.hasClass("is-faded"));
            discussion.setPoll(!element.select("h3 i.fa-align-left").isEmpty());
            discussionList.add(discussion);
        }
        return discussionList;
    } catch (Exception e) {
        Log.e(TAG, "Error fetching URL", e);
        return null;
    }
}
 

@Override
public void addExtraParameters(Connection connection) {
    connection.data("code", giveawayId);
}
 

/**
 * 方法说明：根据绑定的数据type选择绑定数据种类模拟浏览器
 * 
 * @param url           request url
 * @param bindData      bind data
 * @param requestType   request type: "headers" "data" "cookies" etc.
 * @return              Document object.
 * @throws Exception    Exception
 */
public static Document getDocument(String url, Map<String, String> bindData, String requestType) throws Exception {

    Document doc = null;
    Connection conn = null;
    StringWriter strWriter = new StringWriter();
    PrintWriter prtWriter = new PrintWriter(strWriter);

    // En:get max retry count from properties file(com-constants.properties)
    // Jp:プロパティファイルでロックタイムアウトのリトライ回数を取得する Zh:通过properties获取最大retry次数
    int maxRetry = Integer.parseInt(PropertyReader.getProperties(SystemConstants.COM_CONSTANTS)
            .getProperty(UtilsConstants.MAX_RETRY_COUNT));
    // En: get sleep time from properties file Jp:プロパティファイルでロックタイムアウトのスリープ時間を取得する
    int sleepTime = Integer.parseInt(PropertyReader.getProperties(SystemConstants.COM_CONSTANTS)
            .getProperty(UtilsConstants.SLEEP_TIME_COUNT));
    
    int temp = 0;

    // En: if exception is occurred then retry loop is continue to run;
    // Jp: 異常を起きる場合、ループを続き実行する。
    for (int j = 1; j <= maxRetry; j++) {

        try {
            if (j != 1) {
                Thread.sleep(sleepTime);
            }
            temp = Integer.parseInt(String.valueOf(Math.round(Math.random() * (UserAgent.length - 1))));
            conn = Jsoup.connect(url).timeout(10000)
                   // .userAgent(
                            // add userAgent. TODO There is a plan to configure userAgent to load that userAgent from a property file.
                     //       "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.122 Safari/534.30");
                    .userAgent(UserAgent[temp]);
            if (bindData != null && bindData.size() != 0 && !StringUtil.isEmpty(requestType)) {
                switch (requestType) {
                case UtilsConstants.REQUEST_HEADERS:
                    // adds each of the supplied headers to the request. // bluetata 2017/03/22 add
                    conn.headers(bindData);
                    break;
                case UtilsConstants.REQUEST_DATA:
                    // adds all of the supplied data to the request data parameters. // 20170320 bluetata add
                    conn.data(bindData);
                    break;
                case UtilsConstants.REQUEST_COOKIES:
                    // adds each of the supplied cookies to the request. // bluetata 2017/03/22 add
                    conn.cookies(bindData);
                    break;
                default:
                    // TODO stream etc. logic is adding. bluetata 2017/03/22 add
                    break;
                }
            }
            doc = conn.get();

            // En: normal finish situation,loop is broken.
            // Jp: サービスが正常に終了した場合、ループを中止します。
            // Zh: 正常终了的情况、终止循环。
            break;

        } catch (Exception ex) {
            // if throw new Exception(ex); dead code is occurred, retry is invalid.

            // StackTraceを文字列で取得
            ex.printStackTrace(prtWriter);
            String stackTrace = strWriter.toString();

            if (strWriter != null) {
                try {
                    strWriter.close();
                } catch (IOException ioe) {
                    ioe.printStackTrace();
                }
            }
            if (prtWriter != null) {
                prtWriter.close();
            }

            // En:info log is output. Jp: Infoログとして、エラー内容を出力。 Zh:输出到info log。
            Log4jUtil.error("第" + j + "次请求异常。");
            Log4jUtil.error(stackTrace);
        }
    }
    return doc;
}
 

public static String loadPageSource(PageRequest pageRequest) {
    if (!UrlUtil.isUrl(pageRequest.getUrl())) {
        return null;
    }
    try {
        // 请求设置
        Connection conn = Jsoup.connect(pageRequest.getUrl());
        if (pageRequest.getParamMap() != null && !pageRequest.getParamMap().isEmpty()) {
            conn.data(pageRequest.getParamMap());
        }
        if (pageRequest.getCookieMap() != null && !pageRequest.getCookieMap().isEmpty()) {
            conn.cookies(pageRequest.getCookieMap());
        }
        if (pageRequest.getHeaderMap()!=null && !pageRequest.getHeaderMap().isEmpty()) {
            conn.headers(pageRequest.getHeaderMap());
        }
        if (pageRequest.getUserAgent()!=null) {
            conn.userAgent(pageRequest.getUserAgent());
        }
        if (pageRequest.getReferrer() != null) {
            conn.referrer(pageRequest.getReferrer());
        }
        conn.timeout(pageRequest.getTimeoutMillis());
        conn.validateTLSCertificates(pageRequest.isValidateTLSCertificates());
        conn.maxBodySize(0);    // 取消默认1M限制

        // 代理
        if (pageRequest.getProxy() != null) {
            conn.proxy(pageRequest.getProxy());
        }

        conn.ignoreContentType(true);
        conn.method(pageRequest.isIfPost()?Connection.Method.POST:Connection.Method.GET);

        // 发出请求
        Connection.Response resp = conn.execute();
        String pageSource = resp.body();
        return pageSource;
    } catch (IOException e) {
        logger.error(e.getMessage(), e);
        return null;
    }
}
 

/**
 * 加载页面
 *
 * @param pageRequest
 *
 * @return Document
 */
public static Document load(PageRequest pageRequest) {
    if (!UrlUtil.isUrl(pageRequest.getUrl())) {
        return null;
    }
    try {
        // 请求设置
        Connection conn = Jsoup.connect(pageRequest.getUrl());
        if (pageRequest.getParamMap() != null && !pageRequest.getParamMap().isEmpty()) {
            conn.data(pageRequest.getParamMap());
        }
        if (pageRequest.getCookieMap() != null && !pageRequest.getCookieMap().isEmpty()) {
            conn.cookies(pageRequest.getCookieMap());
        }
        if (pageRequest.getHeaderMap()!=null && !pageRequest.getHeaderMap().isEmpty()) {
            conn.headers(pageRequest.getHeaderMap());
        }
        if (pageRequest.getUserAgent()!=null) {
            conn.userAgent(pageRequest.getUserAgent());
        }
        if (pageRequest.getReferrer() != null) {
            conn.referrer(pageRequest.getReferrer());
        }
        conn.timeout(pageRequest.getTimeoutMillis());
        conn.validateTLSCertificates(pageRequest.isValidateTLSCertificates());
        conn.maxBodySize(0);    // 取消默认1M限制

        // 代理
        if (pageRequest.getProxy() != null) {
            conn.proxy(pageRequest.getProxy());
        }

        // 发出请求
        Document html = null;
        if (pageRequest.isIfPost()) {
            html = conn.post();
        } else {
            html = conn.get();
        }
        return html;
    } catch (IOException e) {
        logger.error(e.getMessage(), e);
        return null;
    }
}
 

@Test(expected=IllegalArgumentException.class) public void throwsOnOddData() {
    Connection con = HttpConnection.connect("http://example.com/");
    con.data("Name", "val", "what");
}
 

@Test(expected=IllegalArgumentException.class) public void throwsOnOdddData() {
    Connection con = HttpConnection.connect("http://example.com/");
    con.data("Name", "val", "what");
}