- 简介
通过HttpClient获取网页数据源，通过Jsoup解析数据。先模拟登录，再获取信息。模拟浏览器正常操作，封装请求头信息获取SESSIONID。模拟登录成功后切勿断开会话，依赖登录请求得到的Cookie进行二次请求。请求信息时需打开谷歌浏览器或Fiddler抓包查看参数及请求头信息。
- Maven依赖
<dependency>
<groupId>com.baidu.aip</groupId>
<artifactId>java-sdk</artifactId>
<version>4.8.0</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.2</version>
</dependency>
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>1.11.3</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.4</version>
</dependency>
<dependency>
<groupId>org.json</groupId>
<artifactId>json</artifactId>
<version>20160810</version>
</dependency>
- 基本步骤
1.获取exponent、modulus生成公钥进行密码加密
2.爬虫爬取csrftoken校验
3.添加Post参数模拟浏览器登录获取Cookie(SESSIONID）
4.二次请求
- 工具类
public class B64 {
public static String b64map="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
private static char b64pad = '=';
private static String hexCode = "0123456789abcdef";
// 获取对应16进制字符
public static char int2char(int a){
return hexCode.charAt(a);
}
// Base64转16进制
public static String b64tohex(String s) {
String ret = "";
int k = 0;
int slop = 0;
for(int i = 0; i < s.length(); ++i) {
if(s.charAt(i) == b64pad) break;
int v = b64map.indexOf(s.charAt(i));
if(v < 0) continue;
if(k == 0) {
ret += int2char(v >> 2);
slop = v & 3;
k = 1;
}
else if(k == 1) {
ret += int2char((slop << 2) | (v >> 4));
slop = v & 0xf;
k = 2;
}
else if(k == 2) {
ret += int2char(slop);
ret += int2char(v >> 2);
slop = v & 3;
k = 3;
}
else {
ret += int2char((slop << 2) | (v >> 4));
ret += int2char(v & 0xf);
k = 0;
}
}
if(k == 1)
ret += int2char(slop << 2);
return ret;
}
// 16进制转Base64
public static String hex2b64(String h) {
int i , c;
StringBuilder ret = new StringBuilder();
for(i = 0; i+3 <= h.length(); i+=3) {
c = parseInt(h.substring(i,i+3),16);
ret.append(b64map.charAt(c >> 6));
ret.append(b64map.charAt(c & 63));
}
if(i+1 == h.length()) {
c = parseInt(h.substring(i,i+1),16);
ret.append(b64map.charAt(c << 2));
}
else if(i+2 == h.length()) {
c = parseInt(h.substring(i,i+2),16);
ret.append(b64map.charAt(c >> 2));
ret.append(b64map.charAt((c & 3) << 4));
}
while((ret.length() & 3) > 0) ret.append(b64pad);
return ret.toString();
}
}
public class RSAEncoder {
private static BigInteger n = null;
private static BigInteger e = null;
public static String RSAEncrypt(String pwd, String nStr, String eStr){
n = new BigInteger(nStr,16);
e = new BigInteger(eStr,16);
BigInteger r = RSADoPublic(pkcs1pad2(pwd,(n.bitLength()+7)>>3));
String sp = r.toString(16);
if((sp.length()&1) != 0 )
sp = "0" + sp;
return sp;
}
private static BigInteger RSADoPublic(BigInteger x){
return x.modPow(e, n);
}
private static BigInteger pkcs1pad2(String s, int n){
if(n < s.length() + 11) { // TODO: fix for utf-8
System.err.println("Message too long for RSAEncoder");
return null;
}
byte[] ba = new byte[n];
int i = s.length()-1;
while(i >= 0 && n > 0) {
int c = s.codePointAt(i--);
if(c < 128) { // encode using utf-8
ba[--n] = new Byte(String.valueOf(c));
}
else if((c > 127) && (c < 2048)) {
ba[--n] = new Byte(String.valueOf((c & 63) | 128));
ba[--n] = new Byte(String.valueOf((c >> 6) | 192));
} else {
ba[--n] = new Byte(String.valueOf((c & 63) | 128));
ba[--n] = new Byte(String.valueOf(((c >> 6) & 63) | 128));
ba[--n] = new Byte(String.valueOf((c >> 12) | 224));
}
}
ba[--n] = new Byte("0");
byte[] temp = new byte[1];
Random rdm = new Random(47L);
while(n > 2) { // random non-zero pad
temp[0] = new Byte("0");
while(temp[0] == 0)
rdm.nextBytes(temp);
ba[--n] = temp[0];
}
ba[--n] = 2;
ba[--n] = 0;
return new BigInteger(ba);
}
}
- 代码分享
import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.cookie.Cookie;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.json.JSONArray;
import org.json.JSONObject;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document; import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
import java.util.List; public class ZFsoft {
private final String LOGIN_URL="http://jwgl.hebtu.edu.cn/xtgl/login_slogin.html?language=zh_CN&_t=";
private final String PUBLICKEY_URL="http://jwgl.hebtu.edu.cn/xtgl/login_getPublicKey.html?time=";
private final String CHECK_SCORE_URL="http://jwgl.hebtu.edu.cn/cjcx/cjcx_cxDgXscj.html?doType=query&gnmkdm=N305005"; private CloseableHttpClient httpClient;
private BasicCookieStore basicCookieStore;
public ZFsoft(){
basicCookieStore=new BasicCookieStore();
httpClient= HttpClients
.custom()
.setDefaultCookieStore(basicCookieStore)
.build();
} /**
* 密码加密 RSA
* @param password
* @return
*/
private String encryp(String password){
//一、获取 exponent modulus 生成公钥
String exponent=null,modulus=null;
HttpGet gpkHttpGet=
new HttpGet(PUBLICKEY_URL+new Date().getTime());
gpkHttpGet.setHeader("Accept","application/json, text/javascript, */*; q=0.01");
gpkHttpGet.setHeader("Accept-Encoding","gzip, deflate");
gpkHttpGet.setHeader("Accept-Language","zh-CN,zh;q=0.9");
gpkHttpGet.setHeader("Connection","keep-alive");
gpkHttpGet.setHeader("Host","jwgl.hebtu.edu.cn");
gpkHttpGet.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36");
gpkHttpGet.setHeader("X-Requested-With","XMLHttpRequest");
CloseableHttpResponse gpkResponse=null;
try {
gpkResponse = httpClient.execute(gpkHttpGet);
if (gpkResponse.getStatusLine().getStatusCode() == 200) {
String emJson = EntityUtils.toString(gpkResponse.getEntity(), "utf8");
JSONObject jsonObject = new JSONObject(emJson);
exponent = jsonObject.getString("exponent");
modulus = jsonObject.getString("modulus");
}
}catch (Exception e){
e.printStackTrace();
}finally {
try {
gpkResponse.close();
} catch (IOException e) {
e.printStackTrace();
}
}
//二、根据公钥进行密码加密
password=RSAEncoder.RSAEncrypt(password,B64.b64tohex(modulus),B64.b64tohex(exponent));
password=B64.hex2b64(password);
return password;
} /**
* 获取Token
* @param timestamp
* @return
*/
private String crawlCsrfToken(String timestamp){
String csrftoken=null;
HttpGet csrftokenHttpGet=
new HttpGet(LOGIN_URL+timestamp);
CloseableHttpResponse csrftokenResponse=null;
try {
csrftokenResponse = httpClient.execute(csrftokenHttpGet);
if (csrftokenResponse.getStatusLine().getStatusCode() == 200) {
Document csrftokenDoc = Jsoup.parse(EntityUtils.toString(csrftokenResponse.getEntity(), "utf8"));
csrftoken = csrftokenDoc
.select(".col-sm-4")
.select(".sl_log_rt")
.select("input[id=csrftoken]")
.first()
.attr("value");
return csrftoken;
}
}catch (Exception e){
e.printStackTrace();
}finally {
try {
csrftokenResponse.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return null;
} /**
* 模拟登录
* @param username
* @param password
* @return
*/
public ZFsoft login(String username,String password){
String timestamp=""+new Date().getTime();
HttpPost loginHttpPost=new HttpPost(LOGIN_URL+timestamp);
loginHttpPost.setHeader("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3");
loginHttpPost.setHeader("Accept-Encoding","gzip, deflate");
loginHttpPost.setHeader("Accept-Language","zh-CN,zh;q=0.9");
loginHttpPost.setHeader("Cache-Control","max-age=0");
loginHttpPost.setHeader("Connection","keep-alive");
loginHttpPost.setHeader("Content-Type","application/x-www-form-urlencoded");
loginHttpPost.setHeader("Host","jwgl.hebtu.edu.cn");
loginHttpPost.setHeader("Origin","http://jwgl.hebtu.edu.cn");
loginHttpPost.setHeader("Upgrade-Insecure-Requests","1");
loginHttpPost.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36");
List<NameValuePair> loginParams=new ArrayList<NameValuePair>();
password=this.encryp(password);
String csrftoken=this.crawlCsrfToken(timestamp);
loginParams.add(new BasicNameValuePair("csrftoken",csrftoken));
loginParams.add(new BasicNameValuePair("yhm",username));
loginParams.add(new BasicNameValuePair("mm",password));
loginParams.add(new BasicNameValuePair("mm",password));
CloseableHttpResponse loginResponse=null;
try {
loginHttpPost.setEntity(new UrlEncodedFormEntity(loginParams, "utf8"));
loginResponse = httpClient.execute(loginHttpPost);
List<Cookie>cookies=basicCookieStore.getCookies();
if(cookies.isEmpty()){
System.out.println("The Cookie Is None.");
}else {
for(Cookie cookie:cookies){ }
}
}catch (Exception e){
e.printStackTrace();
}
return this;
} /**
* 查看成绩
* @param xnm
* @param xqm
* @return
*/
public List<Score> checkScore(String xnm,String xqm){
HttpPost scoreHttpPost=new HttpPost(CHECK_SCORE_URL);
scoreHttpPost.setHeader("Accept","application/json, text/javascript, */*; q=0.01");
scoreHttpPost.setHeader("Accept-Encoding","gzip, deflate");
scoreHttpPost.setHeader("Accept-Language","zh-CN,zh;q=0.9");
scoreHttpPost.setHeader("Content-Type","application/x-www-form-urlencoded;charset=UTF-8");
scoreHttpPost.setHeader("Host","jwgl.hebtu.edu.cn");
scoreHttpPost.setHeader("Origin","http://jwgl.hebtu.edu.cn");
scoreHttpPost.setHeader("Proxy-Connection","keep-alive");
scoreHttpPost.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36");
scoreHttpPost.setHeader("X-Requested-With","XMLHttpRequest");
List<NameValuePair>scoreParams=new ArrayList<NameValuePair>();
scoreParams.add(new BasicNameValuePair("xnm",xnm));
scoreParams.add(new BasicNameValuePair("xqm",xqm));
scoreParams.add(new BasicNameValuePair("_search","false"));
scoreParams.add(new BasicNameValuePair("nd",""+new Date().getTime()));
scoreParams.add(new BasicNameValuePair("queryModel.showCount","100"));
scoreParams.add(new BasicNameValuePair("queryModel.currentPage","1"));
scoreParams.add(new BasicNameValuePair("queryModel.sortName",""));
scoreParams.add(new BasicNameValuePair("queryModel.sortOrder","asc"));
scoreParams.add(new BasicNameValuePair("time","1"));
try {
scoreHttpPost.setEntity(new UrlEncodedFormEntity(scoreParams, "utf8"));
CloseableHttpResponse scoreResponse = httpClient.execute(scoreHttpPost);
if (scoreResponse.getStatusLine().getStatusCode() == 200) {
if (scoreResponse.getEntity() != null) {
String scoreJson = EntityUtils.toString(scoreResponse.getEntity(), "utf8");
JSONObject jsonObject = new JSONObject(scoreJson);
JSONArray jsonArray = jsonObject.getJSONArray("items");
List<Score>scoreList=new ArrayList<Score>();
for (int i = 0; i < jsonArray.length(); ++i) {
JSONObject item = (JSONObject) jsonArray.get(i);
Score score=new Score();
score.setXm(item.getString("xm"));
score.setKcmc(item.getString("kcmc"));
score.setBj(item.getString("bj"));
score.setCj(item.getString("cj"));
String jd = "0";
try {
jd = item.getString("jd");
} catch (Exception e) {
e.printStackTrace();
}
score.setJd(jd);
score.setJgmc(item.getString("jgmc"));
score.setKch(item.getString("kch"));
score.setKcxzmc(item.getString("kcxzmc"));
score.setKsxz(item.getString("ksxz"));
scoreList.add(score);
}
return scoreList;
}
}
}catch (Exception e){
e.printStackTrace();
}
return null;
}
}
public class Test {
public static void main(String[] args) {
ZFsoft zFsoft=new ZFsoft();
List<Score>scoreList=zFsoft
.login("2016011493","密码")
.checkScore("","");
for(Score score:scoreList){
System.out.println(score);
}
System.out.println(scoreList.size());
}
}
