JAVA——基于HttpClient的获取帆软FineReport报表爬虫DEMO
·
HttpClient封装类:https://shentuzhigang.blog.csdn.net/article/details/104274609
FineReportUtil
package cn.edu.zstu.myzstu.spyder.ezstu;
import cn.edu.zstu.myzstu.utils.consts.Consts;
import cn.edu.zstu.myzstu.utils.httpclient.HttpClientUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import java.util.HashMap;
import java.util.Map;
/**
* @Author ShenTuZhiGang
* @Version 1.0.0
* @Date 2020-07-12 15:20
*/
public class FineReportUtil {
public static String getSessionID(String url, String username, String viewlet) throws Exception {
String sessionid = "";
Map<String, String> header = new HashMap<>();
header.put("User-Agent", Consts.AGENT);
header.put("Host", Consts.EZLHOST);
Map<String, String> params = new HashMap<>();
params.put("viewlet", viewlet);
params.put("__pi__", "true");
params.put("op", "h5");
params.put("username", username);
String htmlText = HttpClientUtils.doGetRequest(url, header, params);
Document doc = Jsoup.parse(htmlText, "UTF-8");
Element element = doc.select("script").get(1);
sessionid = element.toString().split("get")[7].split("'")[1];
return sessionid;
}
}
DEMO
package cn.edu.zstu.myzstu.spyder.ezstu.query;
import cn.edu.zstu.myzstu.model.lib.BookRecord;
import cn.edu.zstu.myzstu.spyder.ezstu.FineReportUtil;
import cn.edu.zstu.myzstu.utils.consts.Consts;
import cn.edu.zstu.myzstu.utils.httpclient.HttpClientUtils;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* @Author ShenTuZhiGang
* @Version 1.0.0
* @Date 2020-07-12 15:18
*/
@Component
public class LibraryQuery {
private static final String ezlUrl = Consts.PROTOCOL + Consts.EZLHOST +"/webroot/decision/view/report";
public Map<String, List<BookRecord>> getBorrowList(String sid) throws Exception{
Map<String, List<BookRecord>> res = new HashMap<>();
List<BookRecord> borrowlist = new ArrayList<>();
List<BookRecord> historylist = new ArrayList<>();
String sessionid = FineReportUtil.getSessionID(ezlUrl,sid,"/yiban/S图书借阅情况查询.cpt");
Map<String, String> header = new HashMap<>();
header.put("User-Agent", Consts.AGENT);
header.put("Host", Consts.EZLHOST);
header.put("sessionID", sessionid);
String htmlText = HttpClientUtils.doPostRequest(ezlUrl, header, inputGetForm(sessionid, "1")).getContent();
//System.out.println(htmlText);
JSONObject resjson = JSONObject.parseObject(htmlText);
JSONArray bookjson = resjson.getJSONObject("pageContent").getJSONArray("detail").getJSONObject(0).getJSONObject("cellData").getJSONArray("rows");
String sname = bookjson.getJSONObject(4).getJSONArray("cells").getJSONObject(2).getString("text");
if(sname.equals("")) {
throw new Exception("201|学号错误");
}
int pagenum = resjson.getInteger("reportTotalPage");
int borrownum = Integer.parseInt(bookjson.getJSONObject(6).getJSONArray("cells").getJSONObject(2).getString("text"));
int size = pagenum == 1 ? bookjson.size() - 5 : bookjson.size();
for(int k = 15; k < size; k ++) {
JSONArray bookinfo = bookjson.getJSONObject(k).getJSONArray("cells");
String bookname = bookinfo.getJSONObject(2).getString("text");
String borrowdate = bookinfo.getJSONObject(5).getString("text");
BookRecord bookRecord = new BookRecord();
bookRecord.setBookName(bookname);
bookRecord.setBorrowDate(borrowdate);
if(borrowlist.size() < borrownum || k - 14 <= 2* borrownum) {
if(borrowlist.size() == 0 || (borrowlist.size() > 0 && (!bookname.equals(borrowlist.get(borrowlist.size() - 1).getBookName()) || !borrowdate.equals(borrowlist.get(borrowlist.size() - 1).getBorrowDate())))) {
borrowlist.add(bookRecord);
}
else continue;
}
else{
historylist.add(bookRecord);
}
}
for(int i = 1; i < pagenum; i ++) {
htmlText = HttpClientUtils.doPostRequest(ezlUrl, header, inputGetForm(sessionid, i+1+"")).getContent();
resjson = JSONObject.parseObject(htmlText);
bookjson = resjson.getJSONObject("pageContent").getJSONArray("detail").getJSONObject(0).getJSONObject("cellData").getJSONArray("rows");
size = i + 1 < pagenum ? bookjson.size() : bookjson.size() - 4;
for(int k = 0; k < size; k ++) {
JSONArray bookinfo = bookjson.getJSONObject(k).getJSONArray("cells");
String bookname = bookinfo.getJSONObject(2).getString("text");
String borrowdate = bookinfo.getJSONObject(5).getString("text");
BookRecord bookRecord = new BookRecord();
bookRecord.setBookName(bookname);
bookRecord.setBorrowDate(borrowdate);
if(borrowlist.size() < borrownum) {
if(borrowlist.size() == 0 || (borrowlist.size() > 0 && (!bookname.equals(borrowlist.get(borrowlist.size() - 1).getBookName()) || !borrowdate.equals(borrowlist.get(borrowlist.size() - 1).getBorrowDate())))) {
borrowlist.add(bookRecord);
}
}
else {
historylist.add(bookRecord);
}
}
}
res.put("borrowlist", borrowlist);
res.put("historylist", historylist);
return res;
}
//填充请求信息
private Map<String,String> inputGetForm(String sessionid, String pagenum) {
Map<String,String > formData = new HashMap<>();
formData.put("toVanCharts", "true");
formData.put("dynamicHyperlink", "true");
formData.put("op", "page_content");
formData.put("cmd", "json");
formData.put("sessionID", sessionid);
formData.put("pn", pagenum);
formData.put("__fr_locale__", "zh_CN");
return formData;
}
}
参考文章
魔乐社区(Modelers.cn) 是一个中立、公益的人工智能社区,提供人工智能工具、模型、数据的托管、展示与应用协同服务,为人工智能开发及爱好者搭建开放的学习交流平台。社区通过理事会方式运作,由全产业链共同建设、共同运营、共同享有,推动国产AI生态繁荣发展。
更多推荐


所有评论(0)