爬虫日记
案例1:抓取国家药品监督管理局的药店数据
这个官网上的数据防爬机制更新的很快,网上搜索的很多招都不好使了,最后尝试成功
网址:http://app1.nmpa.gov.cn/data_nmpa/face3/dir.html
抓包发现请求参数经过了处理,应该是加密了,不知道加密方式,直接请求接口的方式失败;


列表和详情用的一个页面,通过获取详情的id去直接访问页面解析HTML的方式也不行
通过查看元素,发现每一条数据的连接调用了一个js函数,函数中将id进行了加密处理,但是js文件看不出来怎么加密的。 有了新的主意,可以用自动化测试工具Selenium来模拟点击获取到页面内容。

步骤:
- 下载Selenium的driver
Chrome
点击下载chrome的webdriver: http://chromedriver.storage.googleapis.com/index.html
Firefox
Firefox驱动下载地址为:https://github.com/mozilla/geckodriver/releases/
IE
IE浏览器驱动下载地址为:http://selenium-release.storage.googleapis.com/index.html
下载后配置到环境变量,不同的浏览器版本对应不同的driver版本
- 导入maven依赖
| <dependency> <groupId>org.seleniumhq.selenium</groupId> <artifactId>selenium-java</artifactId> <version>3.141.59</version> </dependency> |
- 开始代码
逻辑:第一个任务先爬取药店列表页面中的药店名称、id、链接的js函数,通过输入页码来爬取所有列表页;第二个任务通过列表中的js函数,用selenium执行函数来获取详情页面,爬取药店详情数据。
| package com.ciics.selenium.task;
import com.ciics.selenium.entity.YaoDian; import org.checkerframework.checker.nullness.compatqual.NullableDecl; import org.openqa.selenium.By; import org.openqa.selenium.WebDriver; import org.openqa.selenium.WebElement; import org.openqa.selenium.support.ui.ExpectedCondition; import org.openqa.selenium.support.ui.WebDriverWait;
import java.util.ArrayList; import java.util.List;
/** * @author cxy * @date 2021/1/29 16:29 */ public class GrabYaoDianTask {
public static List<YaoDian> executeByPage(WebDriver driver, int page) throws Exception { WebDriverWait wait = new WebDriverWait(driver,5); String tablePath="/html/body/center/table[1]/tbody/tr[2]/td/table/tbody/tr/td[3]/table/tbody/tr[2]/td/table[3]/tbody/tr/td[2]/div/div/table[2]"; if (page>1){ try { tablePath="/html/body/center/table[1]/tbody/tr[2]/td/table/tbody/tr/td[3]/table/tbody/tr[2]/td/table[3]/tbody/tr/td[2]/div/table[2]"; WebElement pageInput = wait.until(new ExpectedCondition<WebElement>() { @NullableDecl public WebElement apply(@NullableDecl WebDriver webDriver) { return webDriver.findElement(By.name("goInt")); } }); pageInput.clear(); pageInput.sendKeys(String.valueOf(page)); WebElement pageGo = driver.findElement(By.xpath("//input[@src='images/dataanniu_11.gif']")); pageGo.click(); } catch (Exception e) { e.printStackTrace(); driver.navigate().refresh(); } Thread.sleep(3000); } final String finalTablePath = tablePath; WebElement table = null; try { table = wait.until(new ExpectedCondition<WebElement>() { @NullableDecl public WebElement apply(@NullableDecl WebDriver webDriver) { return webDriver.findElement(By.xpath(finalTablePath)); } }); } catch (Exception e) { e.printStackTrace(); WebElement back = driver.findElement(By.xpath("//*[@id=\"tr1p11\"]/td/table/tbody/tr/td[3]")); back.click(); Thread.sleep(1000); WebElement pageInput = wait.until(new ExpectedCondition<WebElement>() { @NullableDecl public WebElement apply(@NullableDecl WebDriver webDriver) { return webDriver.findElement(By.name("goInt")); } }); pageInput.clear(); pageInput.sendKeys(String.valueOf(page)); WebElement pageGo = driver.findElement(By.xpath("//input[@src='images/dataanniu_11.gif']")); pageGo.click(); table = wait.until(new ExpectedCondition<WebElement>() { @NullableDecl public WebElement apply(@NullableDecl WebDriver webDriver) { return webDriver.findElement(By.xpath(finalTablePath)); } }); }
List<WebElement> elements = table.findElements(By.tagName("a")); ArrayList<YaoDian> yaoDians = new ArrayList<YaoDian>(); for (WebElement element : elements) { String href = element.getAttribute("href"); String text = element.getText(); System.out.println(text); YaoDian yaoDian = new YaoDian(); yaoDian.setName(text); yaoDian.setJs_func(href.replace("javascript:","")); yaoDians.add(yaoDian); } return yaoDians; } }
|
| package com.ciics.selenium.task;
import cn.hutool.core.collection.CollectionUtil; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import com.ciics.selenium.entity.YaoDian; import com.ciics.selenium.entity.YaoDianInfo; import com.ciics.selenium.service.YaoDianInfoService; import com.ciics.selenium.service.YaoDianService; import org.checkerframework.checker.nullness.compatqual.NullableDecl; import org.openqa.selenium.*; import org.openqa.selenium.firefox.FirefoxDriver; import org.openqa.selenium.firefox.FirefoxOptions; import org.openqa.selenium.support.ui.ExpectedCondition; import org.openqa.selenium.support.ui.WebDriverWait; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.scheduling.TaskScheduler; import org.springframework.scheduling.annotation.EnableScheduling; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils;
import java.util.List;
/** * @author cxy * @date 2021/1/29 10:26 */ @Component //1.主要用于标记配置类,兼备Component的效果。 @EnableScheduling public class GrabYaoDianDetailTask { public static final String HOME_URL="http://app1.nmpa.gov.cn/data_nmpa/face3/base.jsp?tableId=41&tableName=TABLE41&title=%D2%A9%C6%B7%BE%AD%D3%AA%C6%F3%D2%B5&bcId=152911863995882985662523838679"; public static int pageNum=13267; @Autowired YaoDianService yaoDianService;
@Autowired YaoDianInfoService yaoDianInfoService;
@Value("${page-begin}") private Integer pageBegin;
@Bean public TaskScheduler taskScheduler() { ThreadPoolTaskScheduler taskScheduler = new ThreadPoolTaskScheduler(); taskScheduler.setPoolSize(5); return taskScheduler; }
// @Scheduled(fixedRate = 60*1000*5) public void grab() throws InterruptedException { QueryWrapper<YaoDian> wrapper = new QueryWrapper<YaoDian>(); wrapper.eq("excute_status",0); Page<YaoDian> page = new Page<YaoDian>(pageBegin,100); IPage<YaoDian> yaoDianIPage = yaoDianService.page(page, wrapper); List<YaoDian> records = yaoDianIPage.getRecords(); if (CollectionUtils.isEmpty(records)){ return; } System.setProperty("webdriver.gecko.driver", "D:\\geckodriver/geckodriver.exe"); FirefoxOptions firefoxOptions = new FirefoxOptions(); firefoxOptions.setPageLoadStrategy(PageLoadStrategy.EAGER); WebDriver driver = new FirefoxDriver(firefoxOptions); //到药品查询首页 try { driver.get(HOME_URL); Thread.sleep(1500); WebDriverWait wait = new WebDriverWait(driver,5); for (YaoDian yaoDian: records) { Thread.sleep(1000); try { ((JavascriptExecutor) driver).executeScript(yaoDian.getJs_func()); } catch (Exception e) { e.printStackTrace(); continue; } Thread.sleep(1000); WebElement contentElement = null; try { contentElement = wait.until(new ExpectedCondition<WebElement>() { @NullableDecl public WebElement apply(@NullableDecl WebDriver webDriver) { return webDriver.findElement(By.xpath("//*[@id=\"content\"]/div/div/table[1]")); } }); } catch (Exception e) { e.printStackTrace(); WebElement back = driver.findElement(By.xpath("//*[@id=\"tr1p11\"]/td/table/tbody/tr/td[3]")); back.click(); Thread.sleep(1000); try { ((JavascriptExecutor) driver).executeScript(yaoDian.getJs_func()); } catch (Exception e1) { e1.printStackTrace(); continue; } try { contentElement = wait.until(new ExpectedCondition<WebElement>() { @NullableDecl public WebElement apply(@NullableDecl WebDriver webDriver) { return webDriver.findElement(By.xpath("//*[@id=\"content\"]/div/div/table[1]")); } }); } catch (Exception exception) { exception.printStackTrace(); ((JavascriptExecutor) driver).executeScript("sform(1,11,this,'41','152911863995882985662523838679')"); Thread.sleep(1000); ((JavascriptExecutor) driver).executeScript(yaoDian.getJs_func()); Thread.sleep(500); ((JavascriptExecutor) driver).executeScript(yaoDian.getJs_func()); Thread.sleep(500); try { contentElement = wait.until(new ExpectedCondition<WebElement>() { @NullableDecl public WebElement apply(@NullableDecl WebDriver webDriver) { return webDriver.findElement(By.xpath("//*[@id=\"content\"]/div/div/table[1]")); } }); } catch (Exception ex) { ex.printStackTrace(); continue; } } } List<WebElement> tds = contentElement.findElements(By.tagName("td")); try { if (CollectionUtil.isNotEmpty(tds)){ YaoDianInfo dto = new YaoDianInfo(); dto.setCode(tds.get(2).getText()); dto.setBusinessRange(tds.get(18).getText()); dto.setBusinessStyle(tds.get(16).getText()); dto.setFazheng_date(tds.get(20).getText()); dto.setFazheng_jigou(tds.get(24).getText()); dto.setLeader(tds.get(12).getText()); dto.setLegalRepresentative(tds.get(10).getText()); dto.setName(tds.get(4).getText()); dto.setQualityLeader(tds.get(14).getText()); dto.setRegisterAddress(tds.get(6).getText()); dto.setWarehouse(tds.get(8).getText()); dto.setZhengshuyouxiaoqi_date(tds.get(22).getText()); dto.setXukezheng_status(tds.get(26).getText()); dto.setYdId(yaoDian.getId()); System.out.println(dto); yaoDianInfoService.saveYaodianInfo(dto); } } catch (Exception e) { e.printStackTrace(); continue; } } } finally { driver.quit(); } }
// @Scheduled(fixedRate = 60*1000*30) public void grabYaoDian() throws Exception { Thread.sleep(3000); //总共40728页 if (pageNum>40728){ return; } //设置driver System.setProperty("webdriver.gecko.driver", "D:\\geckodriver/geckodriver.exe"); //待整个dom树加载完成 FirefoxOptions firefoxOptions = new FirefoxOptions(); firefoxOptions.setPageLoadStrategy(PageLoadStrategy.EAGER); WebDriver driver = new FirefoxDriver(firefoxOptions); //到药品查询首页 driver.get(HOME_URL); Thread.sleep(3000); int max=pageNum+300; try { for (int i=pageNum;i<=max;i++) { try { Thread.sleep(1500); List<YaoDian> yaoDianList = GrabYaoDianTask.executeByPage(driver, i); try { yaoDianService.saveBatch(yaoDianList); } catch (Exception e) { e.printStackTrace(); continue; } } catch (Exception e) { e.printStackTrace(); Thread.sleep(1000); WebElement back = driver.findElement(By.xpath("//*[@id=\"tr1p11\"]/td/table/tbody/tr/td[3]")); back.click(); Thread.sleep(1000); i--; } pageNum=i; } } finally { driver.quit(); } } }
|
因为网址比较卡,页面经常会返回错误页,代码中写了很多重试的逻辑比较乱,最终数据爬取成功。


案例2:爬取某医保中心数据
这个网站比较老,使用的flash技术,数据传输格式是application/amf,爬取接口、解析HTML、selenium爬取页面的方式都不可取,可以使用blazeds来模拟与服务端请求获取数据。
使用Charles抓到的amf包

步骤:
- 导入flex maven坐标,blazeds是flash公司出的一个java客户端工具,一个类似httpclient的客户端,可以模拟发送amf协议的请求,并解析响应。
| <!-- https://mvnrepository.com/artifact/org.apache.flex.blazeds/flex-messaging-core --> <dependency> <groupId>org.apache.flex.blazeds</groupId> <artifactId>flex-messaging-core</artifactId> <version>4.7.2</version> </dependency> <!-- https://mvnrepository.com/artifact/org.apache.flex.blazeds/flex-messaging-common --> <dependency> <groupId>org.apache.flex.blazeds</groupId> <artifactId>flex-messaging-common</artifactId> <version>4.7.2</version> </dependency> |
- 代码编写
| package com.ciics.selenium.example;
import cn.hutool.core.bean.BeanUtil; import cn.hutool.core.bean.copier.CopyOptions; import com.ciics.selenium.entity.YuLinYb; import flex.messaging.io.ArrayCollection; import flex.messaging.io.amf.ASObject; import flex.messaging.io.amf.MessageBody; import flex.messaging.io.amf.client.AMFConnection; import flex.messaging.io.amf.translator.ASTranslator; import flex.messaging.messages.RemotingMessage; import org.springframework.beans.BeanUtils; import org.zreport.crud.entity.RequestEntity;
import java.util.ArrayList; import java.util.HashMap; import java.util.Map; import java.util.UUID;
/** * @author cxy * @date 2021/3/1 16:38 */ public class FlashExample {
public static void main(String[] args) throws Exception { String url = "****/messagebroker/amf"; RemotingMessage remotingMessage = getRemotingMessage(); AMFConnection connection = new AMFConnection(); connection.addHttpRequestHeader("Content-Type", "application/x-amf"); connection.connect(url); connection.addAmfHeader("DSId", UUID.randomUUID().toString().replaceAll("-", "")); // call 方法的 command 对应请求中的 Target Object call = connection.call(null, remotingMessage); // 解析返回值 // flex.messaging.messages.AcknowledgeMessage flex.messaging.messages.AcknowledgeMessage message = (flex.messaging.messages.AcknowledgeMessage) call; ASObject body = (ASObject) message.getBody(); ArrayCollection datas = (ArrayCollection) body.get("datas"); for (Object data:datas){ System.out.println(data); ASObject obj= (ASObject) data; } }
public static RemotingMessage getRemotingMessage() { // content 数组中的第一个对象 RemotingMessage remotingMessage = new RemotingMessage(); // 对应对象里面的各个参数 remotingMessage.setSource(null); remotingMessage.setOperation("execute"); remotingMessage.setDestination("ylzamfservice"); remotingMessage.setTimeToLive(0); remotingMessage.setTimestamp(0); Map<String, Object> headers = new HashMap<String, Object>(); headers.put("DSId", UUID.randomUUID().toString()); headers.put("DSEndpoint", null); remotingMessage.setHeaders(headers); RequestEntity paramObj = new RequestEntity(); paramObj.setServiceid("cp2query"); paramObj.setLoginname("sddlsjcx"); paramObj.setPassword("3706536a9be8e87f4e58ecf3d348047a"); paramObj.setSessionid("NbHsssskG9iDBJCf6cVCDWHAIs0dPAPDCXtoxLfg8C0idm2saJFM!2051497475!1614583876388"); HashMap<String, Object> map = new HashMap<String, Object>(); map.put("selectSQL","select k.aae072,tr_aab301(k.aab301) as aaa103,to_char(k.aae036, 'yyyymmdd') as aae036,k.akc021,k.aac003,k.aac001,tr_aab004(k.aab001) aac010,k.aac004,k.akc023,k.aka120||decode(k.skc057,null,'','+'||k.skc057)||decode(k.skc058,null,'','+'||k.skc058) aka120,k.akc264,k.akc260,nvl(k.mztclj, 0) mztclj,nvl(k.aae005, '无') aae005,decode(k.bkc224, '000', '否', '001', '是', '否') as bkc224,decode(k.bkc225, '000', '否', '001', '是', '否') as bkc225,tr_akb020(k.akb020) as orgname,nvl(k.akc264, 0) - nvl(k.akc261, 0) - nvl(k.akc262, 0) -nvl(k.skc590,0) jgdfje,k.akc262, k.ake181 yljzjj,k.bkc220 dbbxzf,k.bkc121 dbjjzf,nvl(k.bkc222,0)-nvl(k.bkc228,0) fpjjzf from kcao k where k.skc033 = 'Z' and k.akc021 like '9%' and substr(k.aab301, 0, 4) = '6108' and k.skc191 = '102' and (k.akb020 = '999' or '999'= '999') and k.aac001 =:aac001 and k.akc021 =:akc021 and k.bkc224 =:bkc224 and k.bkc225 =:bkc225 and k.sflb =:sflb and (k.aab301 = '999' or '999'= '999') and (k.aab034 = '999' or '999'= '999') and decode(k.ske801,null,'0','1') like '%' and k.aae036 BETWEEN TO_DATE('20200101', 'YYYYMMDD') AND to_date('20200131235959','YYYYMMDDHH24MISS')"); map.put("page",1); map.put("pagesize",100); map.put("rowcount",0); map.put("aae036_qs","20200101"); map.put("aab301","999"); paramObj.setParams(map); ArrayList<Map> list = new ArrayList<Map>(); list.add(map); paramObj.setDatas(list); Object[] body = new Object[]{paramObj}; remotingMessage.setBody(body); remotingMessage.setClientId(UUID.randomUUID().toString()); remotingMessage.setMessageId(UUID.randomUUID().toString()); return remotingMessage; } }
|
魔乐社区(Modelers.cn) 是一个中立、公益的人工智能社区,提供人工智能工具、模型、数据的托管、展示与应用协同服务,为人工智能开发及爱好者搭建开放的学习交流平台。社区通过理事会方式运作,由全产业链共同建设、共同运营、共同享有,推动国产AI生态繁荣发展。
更多推荐



所有评论(0)