org.jsoup.Connection Java Examples

@Override
protected void onPostExecute(Connection.Response response) {
    if (response != null && response.statusCode() == 200) {
        try {
            Log.v(TAG, "Response to JSON request: " + response.body());
            JSONObject root = new JSONObject(response.body());

            if ("success".equals(root.getString("type"))) {
                Activity activity = getFragment().getActivity();
                activity.setResult(CommonActivity.RESPONSE_SYNC_SUCCESSFUL);
                activity.finish();
                return;
            } else {
                String message = root.getString("msg");
                if (message != null) {
                    Toast.makeText(getFragment().getContext(), message, Toast.LENGTH_SHORT).show();
                    return;
                }
            }

        } catch (JSONException e) {
            Log.e(TAG, "Failed to parse JSON object", e);
        }
    }
    Toast.makeText(getFragment().getContext(), "Could not sync.", Toast.LENGTH_SHORT).show();
}
 

/**
 * 根据av号获取相关信息
 */
private void downloadByAv(String avNumber) throws Exception {
    if (avNumber.isEmpty()){
        sendToastMessage("请输入av号");
    }else {
        sendLogMessage("开始转换URL...\n");
        String url = "https://www.bilibili.com/video/av"+avNumber;
        Connection.Response response = Jsoup.connect(url).timeout(10000).execute();
        //普通视频不会重定向，番剧会进行重定向
        if (!response.url().toString().equals(url)){
            url = response.url().toString();
        }
        sendLogMessage("转换URL成功\n");
        downloadByUrl(url);
    }
}
 

@Test public void canInterruptDocumentRead() throws IOException, InterruptedException {
    // todo - implement in interruptable channels, so it's immediate
    final String[] body = new String[1];
    Thread runner = new Thread(new Runnable() {
        public void run() {
            try {
                Connection.Response res = Jsoup.connect("http://jsscxml.org/serverload.stream")
                    .timeout(15 * 1000)
                    .execute();
                body[0] = res.parse().text();
            } catch (IOException e) {
                throw new RuntimeException(e);
            }

        }
    });

    runner.start();
    Thread.sleep(1000 * 7);
    runner.interrupt();
    assertTrue(runner.isInterrupted());
    runner.join();

    assertTrue(body[0].length() > 0);
}
 

@Test
public void handles200WithNoContent() throws IOException {
    Connection con = Jsoup
        .connect("http://direct.infohound.net/tools/200-no-content.pl")
        .userAgent(browserUa);
    Connection.Response res = con.execute();
    Document doc = res.parse();
    assertEquals(200, res.statusCode());

    con = Jsoup
        .connect("http://direct.infohound.net/tools/200-no-content.pl")
        .parser(Parser.xmlParser())
        .userAgent(browserUa);
    res = con.execute();
    doc = res.parse();
    assertEquals(200, res.statusCode());
}
 

@Override
public Document getNextPage(Document page) throws IOException {
    this.currAlbum.pageIndex++;
    boolean endOfAlbum = currAlbum.pageIndex > currAlbum.numPages;
    boolean noMoreSubalbums = albums.isEmpty();
    if (endOfAlbum && noMoreSubalbums){
        throw new IOException("No more pages");
    }
    try {
        Thread.sleep(WAIT_BEFORE_NEXT_PAGE);
    } catch (InterruptedException e) {
        LOGGER.info("Interrupted while waiting before getting next page");
    }
    if (endOfAlbum){
        LOGGER.info("Turning to next album " + albums.get(0).baseURL);
        return getFirstPage();
    } else {
        LOGGER.info("Turning to page " + currAlbum.pageIndex +
                " of album " + currAlbum.baseURL);
        Connection.Response resp = Http.url(currAlbum.getCurrPageURL()).response();
        currAlbum.cookies = resp.cookies();
        currAlbum.currPage = resp.parse();
        return currAlbum.currPage;
    }
}
 

private void attemptLogin() {
    try {
        ConnectionProvider connectionProvider = Inject.connectionProvider();
        Connection.Response response = connectionProvider
                .loginConnection(username, password)
                .execute();

        String cookie = response.cookie("user");
        String cfduid = response.cookie("_cfduid");

        if (!TextUtils.isEmpty(cookie)) {
            subscriber.onNext(new Login(username, cookie, Login.Status.SUCCESSFUL));
        } else {
            subscriber.onNext(new Login(username, null, Login.Status.WRONG_CREDENTIALS));
        }

    } catch (IOException e) {
        subscriber.onError(e);
    }
}
 

public static String get(String url, Map<String, String> headers,
		Map<String, String> params) {
	try {
		Connection connect = Jsoup.connect(url);
		connect.ignoreContentType(true);
		connect.ignoreHttpErrors(true);
		if (params != null) {
			connect.data(params);
		}
		if (headers != null) {
			connect.headers(headers);
		}
		return connect.execute().body();
	} catch (Throwable e) {
	}
	return null;
}
 

@Test
public void maxBodySize() throws IOException {
    String url = "http://direct.infohound.net/tools/large.html"; // 280 K

    Connection.Response defaultRes = Jsoup.connect(url).execute();
    Connection.Response smallRes = Jsoup.connect(url).maxBodySize(50 * 1024).execute(); // crops
    Connection.Response mediumRes = Jsoup.connect(url).maxBodySize(200 * 1024).execute(); // crops
    Connection.Response largeRes = Jsoup.connect(url).maxBodySize(300 * 1024).execute(); // does not crop
    Connection.Response unlimitedRes = Jsoup.connect(url).maxBodySize(0).execute();

    int actualString = 280735;
    assertEquals(actualString, defaultRes.body().length());
    assertEquals(50 * 1024, smallRes.body().length());
    assertEquals(200 * 1024, mediumRes.body().length());
    assertEquals(actualString, largeRes.body().length());
    assertEquals(actualString, unlimitedRes.body().length());

    int actualDocText = 269541;
    assertEquals(actualDocText, defaultRes.parse().text().length());
    assertEquals(49165, smallRes.parse().text().length());
    assertEquals(196577, mediumRes.parse().text().length());
    assertEquals(actualDocText, largeRes.parse().text().length());
    assertEquals(actualDocText, unlimitedRes.parse().text().length());
}
 

private static String _getContent(String url) {
    Connection conn = Jsoup.connect(url)
            .header("Accept", ACCEPT)
            .header("Accept-Encoding", ENCODING)
            .header("Accept-Language", LANGUAGE)
            .header("Connection", CONNECTION)
            .header("Referer", REFERER)
            .header("Host", HOST)
            .header("User-Agent", USER_AGENT)
            .timeout(1000)
            .ignoreContentType(true);
    String html = "";
    try {
        html = conn.post().html();
        html = html.replaceAll("[\n\r]", "");
    }catch (Exception e){
        LOGGER.error("获取URL：" + url + "页面出错", e);
    }
    return html;
}
 

public static Connection.Response doPostFileRequest(String url, String key, InputStream inputStream) throws IOException {
    // Https请求
    if (url.startsWith("https")) {
        trustEveryone();
    }

    Map<String, String> data = new HashMap<>();
    data.put("apiType", "ali,juejin,Huluxia,Imgbb");

    String filename = UUID.randomUUID() + ".jpg";

    return Jsoup.connect(url)
            .method(Connection.Method.POST)
            .ignoreContentType(true)
            .timeout(120000)
            .data(key, filename, inputStream)
            .data(data)
            .execute();
}
 

@Override
public void run() {
    String errorMsg;
    try {
        Connection.Response response = doPostFileRequest(URL, KEY, mStream);
        if (response != null && response.statusCode() == ConstantUtil.HTTP_STATUS_200) {
            Gson gson = new Gson();
            UploadImageResponse entity = gson.fromJson(response.body(), UploadImageResponse.class);
            if (UploadImageResponse.CODE_SUCCESS.equals(entity.getCode())) {
                if (entity.getData() != null && entity.getData().getUrl() != null && !TextUtils.isEmpty(entity.getData().getUrl().getDistribute())) {
                    successOnUI(entity.getData().getUrl().getDistribute());
                    return;
                }
                errorMsg = "图片上传返回数据有误";
            } else {
                errorMsg = entity.getMsg();
            }
        } else {
            errorMsg = "图片上传失败";
        }
    } catch (Exception e) {
        e.printStackTrace();
        errorMsg = e.getMessage();
    }
    failedOnUI(errorMsg);
}
 

private void setupFromConnection(HttpURLConnection conn, Connection.Response previousResponse) throws IOException {
    method = Method.valueOf(conn.getRequestMethod());
    url = conn.getURL();
    statusCode = conn.getResponseCode();
    statusMessage = conn.getResponseMessage();
    contentType = conn.getContentType();

    Map<String, List<String>> resHeaders = createHeaderMap(conn);
    processResponseHeaders(resHeaders);

    // if from a redirect, map previous response cookies into this response
    if (previousResponse != null) {
        for (Map.Entry<String, String> prevCookie : previousResponse.cookies().entrySet()) {
            if (!hasCookie(prevCookie.getKey()))
                cookie(prevCookie.getKey(), prevCookie.getValue());
        }
    }
}
 

@Test public void handlesEscapedRedirectUrls() throws IOException {
    String url = "http://www.altalex.com/documents/news/2016/12/06/questioni-civilistiche-conseguenti-alla-depenalizzazione";
    // sends: Location:http://shop.wki.it/shared/sso/sso.aspx?sso=&url=http%3a%2f%2fwww.altalex.com%2fsession%2fset%2f%3freturnurl%3dhttp%253a%252f%252fwww.altalex.com%253a80%252fdocuments%252fnews%252f2016%252f12%252f06%252fquestioni-civilistiche-conseguenti-alla-depenalizzazione
    // then to: http://www.altalex.com/session/set/?returnurl=http%3a%2f%2fwww.altalex.com%3a80%2fdocuments%2fnews%2f2016%2f12%2f06%2fquestioni-civilistiche-conseguenti-alla-depenalizzazione&sso=RDRG6T684G4AK2E7U591UGR923
    // then : http://www.altalex.com:80/documents/news/2016/12/06/questioni-civilistiche-conseguenti-alla-depenalizzazione

    // bug is that jsoup goes to
    // 	GET /shared/sso/sso.aspx?sso=&url=http%253a%252f%252fwww.altalex.com%252fsession%252fset%252f%253freturnurl%253dhttp%25253a%25252f%25252fwww.altalex.com%25253a80%25252fdocuments%25252fnews%25252f2016%25252f12%25252f06%25252fquestioni-civilistiche-conseguenti-alla-depenalizzazione HTTP/1.1
    // i.e. double escaped

    Connection.Response res = Jsoup.connect(url)
            .proxy("localhost", 8888)
            .execute();
    Document doc = res.parse();
    assertEquals(200, res.statusCode());
}
 

public Connection data(String... keyvals) {
    Validate.notNull(keyvals, "Data key value pairs must not be null");
    Validate.isTrue(keyvals.length %2 == 0, "Must supply an even number of key value pairs");
    for (int i = 0; i < keyvals.length; i += 2) {
        String key = keyvals[i];
        String value = keyvals[i+1];
        Validate.notEmpty(key, "Data key must not be empty");
        Validate.notNull(value, "Data value must not be null");
        req.data(KeyVal.create(key, value));
    }
    return this;
}
 

@Override
public void run() {
    String xsrf = getXsrf();

    Map<String, String> cookies = getCookies();
    if (!cookies.containsKey(ConstantUtil.KEY_XSRF)) {
        cookies.put(ConstantUtil.KEY_XSRF, xsrf);
    }

    Map<String, String> headers = new HashMap<>();
    headers.put("Upgrade-Insecure-Requests", "1");
    headers.put("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3");
    headers.put("Accept-Encoding", "gzip, deflate");

    String number = profile.getNumber().replaceAll("[^0-9]", "");

    String url = String.format(type == TYPE_BLOCK ? ConstantUtil.BLOCK_USER_BASE_URL : ConstantUtil.UNBLOCK_USER_BASE_URL, number);

    try {
        Jsoup.connect(url).headers(headers).cookies(cookies).method(Connection.Method.GET).execute();

        String userProfileUrl = String.format(ConstantUtil.USER_PROFILE_BASE_URL, profile.getUsername());

        new GetUserProfileTask(userProfileUrl, new OnResponseListener<UserProfile>() {
            @Override
            public void onSucceed(UserProfile data) {
                successOnUI((type == TYPE_BLOCK) == data.isBlocked());
            }

            @Override
            public void onFailed(String msg) {
                failedOnUI(msg);
            }
        }).run();
    } catch (IOException e) {
        e.printStackTrace();
        failedOnUI(e.getMessage());
    }
}
 

public static Connection.Response getResponse(String url) throws IOException {
    return Jsoup.connect(url)
            .userAgent("Mozilla/5.0 (Windows NT 10.0; WOW64; rv:54.0) Gecko/20100101 Firefox/54.0")
            .referrer("http://www.google.com")
            .timeout(10000)
            .sslSocketFactory(socketFactory())
            .execute();
}
 

@Test
public void inWildUtfRedirect2() throws IOException {
    Connection.Response res = Jsoup.connect("https://ssl.souq.com/sa-en/2724288604627/s").execute();
    Document doc = res.parse();
    assertEquals(
        "https://saudi.souq.com/sa-en/%D8%AE%D8%B2%D9%86%D8%A9-%D8%A2%D9%85%D9%86%D8%A9-3-%D8%B7%D8%A8%D9%82%D8%A7%D8%AA-%D8%A8%D9%86%D8%B8%D8%A7%D9%85-%D9%82%D9%81%D9%84-%D8%A5%D9%84%D9%83%D8%AA%D8%B1%D9%88%D9%86%D9%8A-bsd11523-6831477/i/?ctype=dsrch",
        doc.location()
    );
}
 

public static String getContent(String url) {
    LOGGER.debug("url:" + url);
    Connection conn = Jsoup.connect(url)
            .header("Accept", ACCEPT)
            .header("Accept-Encoding", ENCODING)
            .header("Accept-Language", LANGUAGE)
            .header("Connection", CONNECTION)
            .header("Referer", REFERER)
            .header("Host", HOST)
            .header("User-Agent", USER_AGENTS.get(uac.incrementAndGet() % USER_AGENTS.size()))
            .header("X-Forwarded-For", getRandomIp())
            .header("Proxy-Client-IP", getRandomIp())
            .header("WL-Proxy-Client-IP", getRandomIp())
            .ignoreContentType(true);
    String html = "";
    try {
        html = conn.post().html();
    }catch (Exception e){
        if(e instanceof  HttpStatusException) {
            HttpStatusException ex = (HttpStatusException) e;
            LOGGER.error("error code:"+ex.getStatusCode());
            if(ex.getStatusCode()==404){
                return "404";
            }
        }
        LOGGER.error("获取URL："+url+" 页面出错", e);
    }
    return html;
}
 

@Test public void usesOnForCheckboxValueIfNoValueSet() {
    Document doc = Jsoup.parse("<form><input type=checkbox checked name=foo></form>");
    FormElement form = (FormElement) doc.select("form").first();
    List<Connection.KeyVal> data = form.formData();
    assertEquals("on", data.get(0).value());
    assertEquals("foo", data.get(0).key());
}
 

private static HttpURLConnection createConnection(Connection.Request req) throws IOException {
    final HttpURLConnection conn = (HttpURLConnection) (
        req.proxy() == null ?
        req.url().openConnection() :
        req.url().openConnection(req.proxy())
    );

    conn.setRequestMethod(req.method().name());
    conn.setInstanceFollowRedirects(false); // don't rely on native redirection support
    conn.setConnectTimeout(req.timeout());
    conn.setReadTimeout(req.timeout());

    if (conn instanceof HttpsURLConnection) {
        if (!req.validateTLSCertificates()) {
            initUnSecureTSL();
            ((HttpsURLConnection)conn).setSSLSocketFactory(sslSocketFactory);
            ((HttpsURLConnection)conn).setHostnameVerifier(getInsecureVerifier());
        }
    }

    if (req.method().hasBody())
        conn.setDoOutput(true);
    if (req.cookies().size() > 0)
        conn.addRequestProperty("Cookie", getRequestCookieString(req));
    for (Map.Entry<String, String> header : req.headers().entrySet()) {
        conn.addRequestProperty(header.getKey(), header.getValue());
    }
    return conn;
}
 

@Override
public void run() {

    String xsrf = getXsrf();

    Map<String, String> headers = new HashMap<>();
    headers.put("Accept", "application/json, text/javascript, */*; q=0.01");

    Map<String, String> datas = new HashMap<>();
    datas.put(ConstantUtil.KEY_XSRF, xsrf);

    Map<String, String> cookies = getCookies();
    if (!cookies.containsKey(ConstantUtil.KEY_XSRF)) {
        cookies.put(ConstantUtil.KEY_XSRF, xsrf);
    }

    try {
        Connection.Response res = Jsoup.connect(mUrl).cookies(cookies).headers(headers).data(datas).method
                (Connection.Method.GET).execute();
        if (res.statusCode() == ConstantUtil.HTTP_STATUS_200 || res.statusCode() == ConstantUtil.HTTP_STATUS_302) {
            successOnUI("成功");
            return;
        } else if (res.statusCode() == ConstantUtil.HTTP_STATUS_304) {
            failedOnUI("不能收藏自己的主题");
            return;
        }
    } catch (IOException e) {
        e.printStackTrace();
    }

    failedOnUI("失败");
}
 

public Connection.KeyVal data(String key) {
    Validate.notEmpty(key, "Data key must not be empty");
    for (Connection.KeyVal keyVal : request().data()) {
        if (keyVal.key().equals(key))
            return keyVal;
    }
    return null;
}
 

@Test public void bodyAndBytesAvailableBeforeParse() throws IOException {
    Connection.Response res = Jsoup.connect(echoURL).execute();
    String body = res.body();
    assertTrue(body.contains("Environment"));
    byte[] bytes = res.bodyAsBytes();
    assertTrue(bytes.length > 100);

    Document doc = res.parse();
    assertTrue(doc.title().contains("Environment"));
}
 

private boolean isLoginSuccess() {
    Map<String, String> datas = new HashMap<String, String>();
    try {
        Connection.Response before_login = Request.execute(Api.MOBILE_COOKIE_LOGIN, Api.USER_AGENT, Connection.Method.GET);
        Document doc = before_login.parse();
        String formhash = doc.getElementById("formhash").attr("value");
        String loginaction = doc.getElementsByTag("form").attr("action");
        Log.d("loginaction", loginaction);

        datas.put("formhash", formhash);
        datas.put("referer", Api.HOST);
        datas.put("fastloginfield", "username");
        datas.put("username", username);
        datas.put("password", password);
        datas.put("questionid", "0");
        datas.put("submit", "登陆");
        datas.put("answer", "");

        Map<String, String> cookies = before_login.cookies();
        Connection.Response logining = Request.execute(Api.HOST + loginaction, Api.USER_AGENT, datas, cookies, Connection.Method.POST);
        if (logining.cookie("v3hW_2132_auth") == null) {
            return false;
        } else {
            cookies = logining.cookies();
            cookies.put("v3hW_2132_saltkey", before_login.cookie("v3hW_2132_saltkey"));
            cookies.put(Athority.PREF_HAS_LOGINED, "yes");
            Athority.addCookies(cookies);
            return true;
        }
    } catch (IOException e) {
        e.printStackTrace();
    }
    return false;
}
 

@Test
public void redirectsResponseCookieToNextResponse() throws IOException {
    Connection con = Jsoup.connect("http://direct.infohound.net/tools/302-cookie.pl");
    Connection.Response res = con.execute();
    assertEquals("asdfg123", res.cookie("token")); // confirms that cookies set on 1st hit are presented in final result
    Document doc = res.parse();
    assertEquals("token=asdfg123; uid=jhy", ihVal("HTTP_COOKIE", doc)); // confirms that redirected hit saw cookie
}
 

/**
 * Prepare to submit this form. A Connection object is created with the request set up from the form values. You
 * can then set up other options (like user-agent, timeout, cookies), then execute it.
 * @return a connection prepared from the values of this form.
 * @throws IllegalArgumentException if the form's absolute action URL cannot be determined. Make sure you pass the
 * document's base URI when parsing.
 */
public Connection submit() {
    String action = hasAttr("action") ? absUrl("action") : baseUri();
    Validate.notEmpty(action, "Could not determine a form action URL for submit. Ensure you set a base URI when parsing.");
    Connection.Method method = attr("method").toUpperCase().equals("POST") ?
            Connection.Method.POST : Connection.Method.GET;

    return Jsoup.connect(action)
            .data(formData())
            .method(method);
}
 

public static Connection.Response execute(String url, String agent, Map<String, String> datas, Map<String, String> cookies, Connection.Method method) {
    Connection.Response response = null;
    try {
        response =
                Jsoup.connect(url).userAgent(agent).timeout(10 * 1000).data(datas).cookies(cookies).method(method).execute();
    } catch (IOException e) {
        e.printStackTrace();
    }
    return response;
}
 

@Override
protected List<Giveaway> doInBackground(Void... params) {
    Log.d(TAG, "Fetching giveaways for user " + path + " on page " + page);

    try {
        // Fetch the Giveaway page
        Connection connection = Jsoup.connect("https://www.steamgifts.com/user/" + path + "/search")
                .userAgent(Constants.JSOUP_USER_AGENT)
                .timeout(Constants.JSOUP_TIMEOUT);
        connection.data("page", Integer.toString(page));
        if (SteamGiftsUserData.getCurrent(fragment.getContext()).isLoggedIn()) {
            connection.cookie("PHPSESSID", SteamGiftsUserData.getCurrent(fragment.getContext()).getSessionId());
            connection.followRedirects(false);
        }

        Connection.Response response = connection.execute();
        Document document = response.parse();

        if (response.statusCode() == 200) {

            SteamGiftsUserData.extract(fragment.getContext(), document);

            if (!user.isLoaded())
                foundXsrfToken = Utils.loadUserProfile(user, document);

            // Parse all rows of giveaways
            return Utils.loadGiveawaysFromList(document);
        } else {
            Log.w(TAG, "Got status code " + response.statusCode());
            return null;
        }
    } catch (Exception e) {
        Log.e(TAG, "Error fetching URL", e);
        return null;
    }
}
 

@Test
public void handlesDodgyCharset() throws IOException {
    // tests that when we get back "UFT8", that it is recognised as unsupported, and falls back to default instead
    String url = "http://direct.infohound.net/tools/bad-charset.pl";
    Connection.Response res = Jsoup.connect(url).execute();
    assertEquals("text/html; charset=UFT8", res.header("Content-Type")); // from the header
    assertEquals(null, res.charset()); // tried to get from header, not supported, so returns null
    Document doc = res.parse(); // would throw an error if charset unsupported
    assertTrue(doc.text().contains("Hello!"));
    assertEquals("UTF-8", res.charset()); // set from default on parse
}
 

@Override
public void run() {

    String xsrf = getXsrf();

    Map<String, String> headers = new HashMap<>();
    headers.put("Accept", "application/json, text/javascript, */*; q=0.01");
    headers.put("X-Requested-With", "XMLHttpRequest");
    headers.put("Content-Type", "application/x-www-form-urlencoded");

    Map<String, String> cookies = getCookies();
    if (!cookies.containsKey(ConstantUtil.KEY_XSRF)) {
        cookies.put(ConstantUtil.KEY_XSRF, xsrf);
    }

    try {
        Connection.Response res = Jsoup.connect(mUrl).cookies(cookies).headers(headers).method
                (Connection.Method.GET).execute();
        if (res.statusCode() == ConstantUtil.HTTP_STATUS_200 || res.statusCode() == ConstantUtil.HTTP_STATUS_302) {
            Gson gson = new Gson();
            BaseResponse response = gson.fromJson(res.body(), BaseResponse.class);
            if (response.getSuccess() == 1) {
                // 赞成功
                successOnUI(true);
            } else {
                // 以前已经赞过该评论
                successOnUI(false);
            }
            return;
        }
    } catch (IOException e) {
        e.printStackTrace();
    }

    failedOnUI("失败");
}