org.jsoup.Connection.Response#cookies

Source File: TwodgalleriesRipper.java From ripme with MIT License

6 votes

private void login() throws IOException {
    Response resp = Http.url(this.url).response();
    cookies = resp.cookies();
    String ctoken = resp.parse().select("form > input[name=ctoken]").first().attr("value");

    Map<String,String> postdata = new HashMap<>();
    postdata.put("user[login]", new String(Base64.decode("cmlwbWU=")));
    postdata.put("user[password]", new String(Base64.decode("cmlwcGVy")));
    postdata.put("rememberme", "1");
    postdata.put("ctoken", ctoken);

    resp = Http.url("http://en.2dgalleries.com/account/login")
               .referrer("http://en.2dgalleries.com/")
               .cookies(cookies)
               .data(postdata)
               .method(Method.POST)
               .response();
    cookies = resp.cookies();
}

Source File: FuskatorRipper.java From ripme with MIT License

5 votes

@Override
public Document getFirstPage() throws IOException {
    // return Http.url(url).get();
    Response res = Http.url(url).response();
    cookies = res.cookies();
    return res.parse();
}

Source File: ThechiveRipper.java From ripme with MIT License

5 votes

@Override
public Document getNextPage(Document doc) throws IOException {
    Matcher matcher = p1.matcher(url.toExternalForm());

    if (matcher.matches()) {
        // url type thechive.com/YEAR/MONTH/DAY/POSTTITLE/ has a single page.
        return null;
    } else {
        if (nextSeed == null) {
            throw new IOException("No more pages.");
        }
    }

    // Following try block checks if the next JSON object has images or not.
    // This is done to avoid IOException in rip() method, caused when
    // getURLsFromPage() returns empty list.
    JSONArray imgList;
    try {
        Response response = Http.url(jsonUrl).data("seed", nextSeed).data("queryType", "by-username")
                .data("username", username).ignoreContentType().cookies(cookies).response();
        cookies = response.cookies();
        JSONObject json = new JSONObject(response.body());
        imgList = json.getJSONArray("uploads");
    } catch (Exception e) {
        throw new IOException("Error fetching next page.", e);
    }

    if (imgList != null && imgList.length() > 0) {
        // Pass empty document as it is of no use for thechive.com/userName url type.
        return new Document(url.toString());
    } else {
        // Return null as this is last page.
        return null;
    }
}

Source File: AmazonLoginApater.java From crawler-jsoup-maven with Apache License 2.0

5 votes

public static void main(String[] args) {
    // grab login form page first
    try {
        
        //lets make data map containing all the parameters and its values found in the form
        Map<String, String> mapParamsData = new HashMap<String, String>();
        mapParamsData.put("email", "[email protected]");
        mapParamsData.put("password", "bluetata");
        
        Response loginResponse = Jsoup.connect("https://passport.jd.com/new/login.aspx")
                .userAgent(USER_AGENT)
                .timeout(TIMEOUT_UNIT * TIMEOUT_TIMES)
                .data(mapParamsData)
                .method(Method.POST)
                .followRedirects(true)
                .execute();
        
        System.out.println("Fetched login page");
        // System.out.println(loginResponse.toString());
        
      //get the cookies from the response, which we will post to the action URL
        Map<String, String> mapLoginPageCookies = loginResponse.cookies();
        
        System.out.println(mapLoginPageCookies);
        
    } catch (Exception e) {
        e.printStackTrace();
    }
}

Source File: WebtoonsRipper.java From ripme with MIT License

4 votes

@Override
public Document getFirstPage() throws IOException {
    Response resp = Http.url(url).response();
    cookies = resp.cookies();
    return Http.url(url).get();
}

Source File: GITHUBLoginApater.java From crawler-jsoup-maven with Apache License 2.0

4 votes

/**
     * @param userName 用户名
     * @param pwd 密码
     * @throws Exception
     */
    public static void simulateLogin(String userName, String pwd) throws Exception {

        /* 
         * 第一次请求 
         * grab login form page first
         * 获取登陆提交的表单信息，及修改其提交data数据（login，password）
         */
        // get the response, which we will post to the action URL(rs.cookies())
        Connection con = Jsoup.connect(LOGIN_URL);  // 获取connection
        con.header(USER_AGENT, USER_AGENT_VALUE);   // 配置模拟浏览器
        Response rs = con.execute();                // 获取响应
        Document d1 = Jsoup.parse(rs.body());       // 转换为Dom树
        
        System.out.println(d1);
        
        List<Element> eleList = d1.select("form");  // 获取提交form表单，可以通过查看页面源码代码得知

        // 获取cooking和表单属性
        // lets make data map containing all the parameters and its values found in the form
        Map<String, String> datas = new HashMap<>();
        
        // 01/24/2019 17:45 bluetata 更新 -------------------------------------------------------------- Start ----------
        // GitHub多次改版更新，最新的提交request data为:
        
        // authenticity_token   ll0RJnG1f9XDAaN1DxnyTDzCs+YXweEZWel9kGkq8TvXH83HjCwPG048sJ/VVjDA94YmbF0qvUgcJx8/IKlP8Q==
        // commit  Sign+in
        // login   bluetata
        // password    password123
        // utf8    ✓
        
        for(int i = 0; i < eleList.size(); i++) {
        
            for (Element e : eleList.get(i).getAllElements()) {
                // 设置用户名
                if (e.attr("name").equals("login")) {
                    e.attr("value", userName);
                }
                // 设置用户密码
                if (e.attr("name").equals("password")) {
                    e.attr("value", pwd);
                }
                // 排除空值表单属性
                if (e.attr("name").length() > 0) {
                    datas.put(e.attr("name"), e.attr("value"));
                }
            }
        }

        
//      旧逻辑  delete  01/24/2019 17:49 bluetata --------------------------------------------start
//        for (Element e : eleList.get(0).getAllElements()) {
//            // 设置用户名
//            if (e.attr("name").equals("login")) {
//                e.attr("value", userName);
//            }
//            // 设置用户密码
//            if (e.attr("name").equals("password")) {
//                e.attr("value", pwd);
//            }
//            // 排除空值表单属性
//            if (e.attr("name").length() > 0) {
//                datas.put(e.attr("name"), e.attr("value"));
//            }
//        }
//      旧逻辑  delete  01/24/2019 17:49 bluetata --------------------------------------------end
        
        
        // 01/24/2019 17:45 bluetata 更新 --------------------------------------------------------------- End -----------
        
        /*
         * 第二次请求，以post方式提交表单数据以及cookie信息
         */
        Connection con2 = Jsoup.connect("https://github.com/session");
        con2.header(USER_AGENT, USER_AGENT_VALUE);
        // 设置cookie和post上面的map数据
        Response login = con2.ignoreContentType(true).followRedirects(true).method(Method.POST).data(datas).cookies(rs.cookies()).execute();
        // 打印，登陆成功后的信息
        System.out.println(login.body());

        // 登陆成功后的cookie信息，可以保存到本地，以后登陆时，只需一次登陆即可
        Map<String, String> map = login.cookies();
        for (String s : map.keySet()) {
            System.out.println(s + " : " + map.get(s));
        }
    }

Source File: CentralnicLoinApater.java From crawler-jsoup-maven with Apache License 2.0

4 votes

public static void main(String[] args) throws Exception {
    
    String url = "https://registrar-console.centralnic.com/dashboard/login";
    String userAgent = "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36";
    
    Response res = Jsoup.connect(url).userAgent(userAgent).timeout(1000 * 3).execute();

    Document d1 = Jsoup.parse(res.body());
    
    List<Element> et = d1.select("#login_form");// 获取form表单
    String XSRF = null;
    // 获取，cooking和表单属性，下面map存放post时的数据
    Map<String, String> datas = new java.util.HashMap<>();
    for (Element e : et.get(0).getAllElements()) {
      if (e.attr("name").equals("regid")) {
        e.attr("value", "*****");// 设置id
      }
      if (e.attr("name").equals("user")) {
        e.attr("value", "*****");// 设置用户名
      }
      if (e.attr("name").equals("password")) {
        e.attr("value", "*****"); // 设置用户密码
      }
      if (e.attr("name").equals("XSRF")) {
        XSRF = e.attr("value");
      }
      if (e.attr("name").length() > 0) {// 排除空值表单属性
        datas.put(e.attr("name"), e.attr("value"));
      }
    }
    
    System.out.println(datas);
    System.out.println("======================");
    System.out.println(res.url());
    System.out.println(res.statusCode());
    
    Map<String, String> cookies = res.cookies();
    cookies.put("remember_me", "H2454482%3Atoddhan");
    cookies.put("lang", "en");
    
    System.out.println(res.cookies());
    
    /* 
     * 第二次请求，以post方式提交表单数据以及cookie信息 
     */  
    Connection con2 = Jsoup.connect("https://registrar-console.centralnic.com/dashboard/login_target");  
    con2.header("User-Agent", userAgent);  
    // 设置cookie和post上面的map数据  
    Response login = con2.ignoreContentType(true).followRedirects(true).method(Method.POST)  
                            .data(datas).cookies(cookies).referrer("https://registrar-console.centralnic.com/dashboard/login")
                            .header("host", "registrar-console.centralnic.com")
                            .execute();  
    
    Map<String, String> cookies2 = login.cookies();
    cookies2.put("remember_me", "H2454482%3Atoddhan");
    cookies2.put("lang", "en");
    System.out.println(cookies2);
    
    System.out.println(login.statusCode());
    
    System.err.println(JsoupUtil.getDocumentWithCookies("https://registrar-console.centralnic.com/", cookies2));
    
    
    System.out.println(JsoupUtil.getDocumentWithCookies("https://registrar-console.centralnic.com/graphs/file_browser/droplist", cookies2));
    
    
}

Source File: JDLoginApater.java From crawler-jsoup-maven with Apache License 2.0

4 votes

public static void main(String[] args) {
    // grab login form page first
    try {
        
        //lets make data map containing all the parameters and its values found in the form
        Map<String, String> mapParamsData = new HashMap<String, String>();
        mapParamsData.put("loginType", "f");
        mapParamsData.put("loginname", "18241141433");
        mapParamsData.put("nloginpwd", "password");
        mapParamsData.put("eid", "2QE5VJVZUBCRYD7LQZBJBTEFRNKPMQQA5OXKXNY7AAN4A3DKDTR7IN3GXHE5C6B4GTMW3Z53B4RGORB6YG5LUWF2UA");
        mapParamsData.put("fp", "ae5baf289624644fced3f921c6a3792c");
        mapParamsData.put("pubKey", "MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDC7kw8r6tq43pwApYvkJ5laljaN9BZb21TAIfT/vexbobzH7Q8SUdP5uDPXEBKzOjx2L28y7Xs1d9v3tdPfKI2LR7PAzWBmDMn8riHrDDNpUpJnlAGUqJG9ooPn8j7YNpcxCa1iybOlc2kEhmJn5uwoanQq+CA6agNkqly2H4j6wIDAQAB");
        mapParamsData.put("sa_token", "B68C442BE645754F33277E701208059080DD726A94A73F76DEC3053A838549C06EB7D3797CE1C5BBE7C2B2EF9CA7D4676F3D489984B517943EA13575FA80C7E73160F85EDB2705D145C52621D18B86C98B49AAF0DA97A2A7A964A78DDDA048AE592CF17C2AD8BF442AD743460B9316993DCDF5924AD8536FD4392C95A998E1C4C4EEDF76BF8FF03AAC145E449EAB889368EE1E7DA72B18881D527D9F51BAD9E2678DDEAFB33A647DD6D48B2A3BE1BC51DDC55AB1EAAEE2DE9A3CEA3702F93AAD1EC8EF740B632F5A4EC102498CDB31AF91CEA15DB3B6DF6FAC6CA31473ACC5E2CD727F80D2746F504A85379E7F3971086C13BA743F21731CEBFEC558E54E8D5D486CC3A19266E238F539A59C2F8630964981217DCC3B31B324F7DBF41FAEA47CA363904F06816BA9730B31BDD9FFA5498C1D0C36D67F315BA4F9236AC77BAFD5");
        mapParamsData.put("seqSid", "5844668515208615000");
        mapParamsData.put("uuid", "5653262a-5ef1-47c6-8ac2-427f519fdcfa");
        
        Response loginResponse = Jsoup.connect(LOGIN_URI)
                .userAgent(USER_AGENT)
                .timeout(TIMEOUT_UNIT * TIMEOUT_TIMES)
                .data(mapParamsData)
                .method(Method.POST)
                .followRedirects(true)
                .execute();
        
        System.out.println("Fetched login page");
        // System.out.println(loginResponse.toString());
        
      //get the cookies from the response, which we will post to the action URL
        Map<String, String> mapLoginPageCookies = loginResponse.cookies();
        
        System.out.println(mapLoginPageCookies);
        
        Document doc = Jsoup.connect("http://order.jd.com/center/list.action")
                .cookies(mapLoginPageCookies)
                .timeout(30000)
                .get();
        System.out.println(doc.toString());
        
    } catch (Exception e) {
        e.printStackTrace();
    }
}

Source File: PostSend2.java From crawler-jsoup-maven with Apache License 2.0

4 votes

public static void main(String[] args) throws Exception {
        
        String url = "http://jzsc.mohurd.gov.cn/dataservice/query/comp/list";
        Response res = Jsoup.connect(url).execute();
        Map<String, String> cookies = res.cookies();
        System.out.println(cookies);
        
        Map<String, String> datas = new HashMap<>();
        datas.put("$total", "297267");
        datas.put("$reload", "1");
        datas.put("$pg", "31&&1=1");
        datas.put("$pgsz", "15");
        
        Response res1 = Jsoup.connect(url).cookies(cookies).data(datas).method(Method.POST).execute();
        
        String html = res1.body();
        System.out.println(html);
        
//        String jsonBody = "{pg:30,ps:15,tt:297267,pn:5,pc:19818,id:'',st:true}"; 
//        
//        Connection connection = Jsoup.connect("http://jzsc.mohurd.gov.cn/dataservice/query/comp/list")  
//        .userAgent("Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36") // User-Agent of Chrome 55  
//        .referrer("http://jzsc.mohurd.gov.cn/dataservice/query/comp/list")  
//        .header("Content-Type", "application/x-www-form-urlencoded")  
//        .header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")  
//        .header("Accept-Encoding", "gzip, deflate")  
//        .header("Accept-Language", "zh-CN,en-US;q=0.8,zh;q=0.6,ja;q=0.4,en;q=0.2")  
//        .header("Cookie", "filter_comp=; JSESSIONID=F53A68FBA45CCF750F9F32ACF090DC9B")  
//        .header("Host", "jzsc.mohurd.gov.cn")  
//        .header("Connection", "keep-alive")  
//        .header("X-Requested-With", "XMLHttpRequest")  
//        .header("Upgrade-Insecure-Requests", "1")  
//        .requestBody(jsonBody)  
//        .maxBodySize(100)  
//        .timeout(1000 * 10)  
//        .method(Connection.Method.POST);  
//  
//        Response response = connection.execute(); 
//        
//        String body = response.body();
//        System.out.println(body);

    }

Java Code Examples for org.jsoup.Connection.Response#cookies()