案例1:抓取国家药品监督管理局的药店数据

这个官网上的数据防爬机制更新的很快,网上搜索的很多招都不好使了,最后尝试成功

网址:http://app1.nmpa.gov.cn/data_nmpa/face3/dir.html

 

抓包发现请求参数经过了处理,应该是加密了,不知道加密方式,直接请求接口的方式失败;

 

 

列表和详情用的一个页面,通过获取详情的id去直接访问页面解析HTML的方式也不行

 

通过查看元素,发现每一条数据的连接调用了一个js函数,函数中将id进行了加密处理,但是js文件看不出来怎么加密的。 有了新的主意,可以用自动化测试工具Selenium来模拟点击获取到页面内容。

步骤:

  1. 下载Selenium的driver

Chrome

点击下载chrome的webdriver: http://chromedriver.storage.googleapis.com/index.html

Firefox

Firefox驱动下载地址为:https://github.com/mozilla/geckodriver/releases/

IE

IE浏览器驱动下载地址为:http://selenium-release.storage.googleapis.com/index.html

下载后配置到环境变量,不同的浏览器版本对应不同的driver版本

  1. 导入maven依赖

<dependency>

            <groupId>org.seleniumhq.selenium</groupId>

            <artifactId>selenium-java</artifactId>

            <version>3.141.59</version>

        </dependency>

  1. 开始代码

逻辑:第一个任务先爬取药店列表页面中的药店名称、id、链接的js函数,通过输入页码来爬取所有列表页;第二个任务通过列表中的js函数,用selenium执行函数来获取详情页面,爬取药店详情数据。

package com.ciics.selenium.task;

 

import com.ciics.selenium.entity.YaoDian;

import org.checkerframework.checker.nullness.compatqual.NullableDecl;

import org.openqa.selenium.By;

import org.openqa.selenium.WebDriver;

import org.openqa.selenium.WebElement;

import org.openqa.selenium.support.ui.ExpectedCondition;

import org.openqa.selenium.support.ui.WebDriverWait;

 

import java.util.ArrayList;

import java.util.List;

 

/**

 * @author cxy

 * @date 2021/1/29 16:29

 */

public class GrabYaoDianTask {

 

 

    public static List<YaoDian> executeByPage(WebDriver driver, int page) throws Exception {

        WebDriverWait wait = new WebDriverWait(driver,5);

        String tablePath="/html/body/center/table[1]/tbody/tr[2]/td/table/tbody/tr/td[3]/table/tbody/tr[2]/td/table[3]/tbody/tr/td[2]/div/div/table[2]";

        if (page>1){

            try {

                tablePath="/html/body/center/table[1]/tbody/tr[2]/td/table/tbody/tr/td[3]/table/tbody/tr[2]/td/table[3]/tbody/tr/td[2]/div/table[2]";

                WebElement pageInput = wait.until(new ExpectedCondition<WebElement>() {

                    @NullableDecl

                    public WebElement apply(@NullableDecl WebDriver webDriver) {

                        return webDriver.findElement(By.name("goInt"));

                    }

                });

                pageInput.clear();

                pageInput.sendKeys(String.valueOf(page));

                WebElement pageGo = driver.findElement(By.xpath("//input[@src='images/dataanniu_11.gif']"));

                pageGo.click();

            } catch (Exception e) {

                e.printStackTrace();

                driver.navigate().refresh();

            }

            Thread.sleep(3000);

        }

        final String finalTablePath = tablePath;

        WebElement table = null;

        try {

            table = wait.until(new ExpectedCondition<WebElement>() {

                @NullableDecl

                public WebElement apply(@NullableDecl WebDriver webDriver) {

                    return webDriver.findElement(By.xpath(finalTablePath));

                }

            });

        } catch (Exception e) {

            e.printStackTrace();

            WebElement back = driver.findElement(By.xpath("//*[@id=\"tr1p11\"]/td/table/tbody/tr/td[3]"));

            back.click();

            Thread.sleep(1000);

            WebElement pageInput = wait.until(new ExpectedCondition<WebElement>() {

                @NullableDecl

                public WebElement apply(@NullableDecl WebDriver webDriver) {

                    return webDriver.findElement(By.name("goInt"));

                }

            });

            pageInput.clear();

            pageInput.sendKeys(String.valueOf(page));

            WebElement pageGo = driver.findElement(By.xpath("//input[@src='images/dataanniu_11.gif']"));

            pageGo.click();

            table = wait.until(new ExpectedCondition<WebElement>() {

                @NullableDecl

                public WebElement apply(@NullableDecl WebDriver webDriver) {

                    return webDriver.findElement(By.xpath(finalTablePath));

                }

            });

        }

 

        List<WebElement> elements = table.findElements(By.tagName("a"));

        ArrayList<YaoDian> yaoDians = new ArrayList<YaoDian>();

        for (WebElement element : elements) {

            String href = element.getAttribute("href");

            String text = element.getText();

            System.out.println(text);

            YaoDian yaoDian = new YaoDian();

            yaoDian.setName(text);

            yaoDian.setJs_func(href.replace("javascript:",""));

            yaoDians.add(yaoDian);

        }

        return yaoDians;

    }

}

 

package com.ciics.selenium.task;

 

import cn.hutool.core.collection.CollectionUtil;

import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;

import com.baomidou.mybatisplus.core.metadata.IPage;

import com.baomidou.mybatisplus.extension.plugins.pagination.Page;

import com.ciics.selenium.entity.YaoDian;

import com.ciics.selenium.entity.YaoDianInfo;

import com.ciics.selenium.service.YaoDianInfoService;

import com.ciics.selenium.service.YaoDianService;

import org.checkerframework.checker.nullness.compatqual.NullableDecl;

import org.openqa.selenium.*;

import org.openqa.selenium.firefox.FirefoxDriver;

import org.openqa.selenium.firefox.FirefoxOptions;

import org.openqa.selenium.support.ui.ExpectedCondition;

import org.openqa.selenium.support.ui.WebDriverWait;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.beans.factory.annotation.Value;

import org.springframework.context.annotation.Bean;

import org.springframework.scheduling.TaskScheduler;

import org.springframework.scheduling.annotation.EnableScheduling;

import org.springframework.scheduling.annotation.Scheduled;

import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler;

import org.springframework.stereotype.Component;

import org.springframework.util.CollectionUtils;

 

import java.util.List;

 

/**

 * @author cxy

 * @date 2021/1/29 10:26

 */

@Component      //1.主要用于标记配置类,兼备Component的效果。

@EnableScheduling

public class GrabYaoDianDetailTask {

    public static final String HOME_URL="http://app1.nmpa.gov.cn/data_nmpa/face3/base.jsp?tableId=41&tableName=TABLE41&title=%D2%A9%C6%B7%BE%AD%D3%AA%C6%F3%D2%B5&bcId=152911863995882985662523838679";

    public static int pageNum=13267;

    @Autowired

    YaoDianService yaoDianService;

 

    @Autowired

    YaoDianInfoService yaoDianInfoService;

 

    @Value("${page-begin}")

    private Integer pageBegin;

 

    @Bean

    public TaskScheduler taskScheduler() {

        ThreadPoolTaskScheduler taskScheduler = new ThreadPoolTaskScheduler();

        taskScheduler.setPoolSize(5);

        return taskScheduler;

    }

 

//    @Scheduled(fixedRate = 60*1000*5)

    public void grab() throws InterruptedException {

        QueryWrapper<YaoDian> wrapper = new QueryWrapper<YaoDian>();

        wrapper.eq("excute_status",0);

        Page<YaoDian> page = new Page<YaoDian>(pageBegin,100);

        IPage<YaoDian> yaoDianIPage = yaoDianService.page(page, wrapper);

        List<YaoDian> records = yaoDianIPage.getRecords();

        if (CollectionUtils.isEmpty(records)){

            return;

        }

        System.setProperty("webdriver.gecko.driver", "D:\\geckodriver/geckodriver.exe");

        FirefoxOptions firefoxOptions = new FirefoxOptions();

        firefoxOptions.setPageLoadStrategy(PageLoadStrategy.EAGER);

        WebDriver driver = new FirefoxDriver(firefoxOptions);

        //到药品查询首页

        try {

            driver.get(HOME_URL);

            Thread.sleep(1500);

            WebDriverWait wait = new WebDriverWait(driver,5);

            for (YaoDian yaoDian: records) {

                Thread.sleep(1000);

                try {

                    ((JavascriptExecutor) driver).executeScript(yaoDian.getJs_func());

                } catch (Exception e) {

                    e.printStackTrace();

                    continue;

                }

                Thread.sleep(1000);

                WebElement contentElement = null;

                try {

                    contentElement = wait.until(new ExpectedCondition<WebElement>() {

                        @NullableDecl

                        public WebElement apply(@NullableDecl WebDriver webDriver) {

                            return webDriver.findElement(By.xpath("//*[@id=\"content\"]/div/div/table[1]"));

                        }

                    });

                } catch (Exception e) {

                    e.printStackTrace();

                    WebElement back = driver.findElement(By.xpath("//*[@id=\"tr1p11\"]/td/table/tbody/tr/td[3]"));

                    back.click();

                    Thread.sleep(1000);

                    try {

                        ((JavascriptExecutor) driver).executeScript(yaoDian.getJs_func());

                    } catch (Exception e1) {

                        e1.printStackTrace();

                        continue;

                    }

                    try {

                        contentElement = wait.until(new ExpectedCondition<WebElement>() {

                            @NullableDecl

                            public WebElement apply(@NullableDecl WebDriver webDriver) {

                                return webDriver.findElement(By.xpath("//*[@id=\"content\"]/div/div/table[1]"));

                            }

                        });

                    } catch (Exception exception) {

                        exception.printStackTrace();

                        ((JavascriptExecutor) driver).executeScript("sform(1,11,this,'41','152911863995882985662523838679')");

                        Thread.sleep(1000);

                        ((JavascriptExecutor) driver).executeScript(yaoDian.getJs_func());

                        Thread.sleep(500);

                        ((JavascriptExecutor) driver).executeScript(yaoDian.getJs_func());

                        Thread.sleep(500);

                        try {

                            contentElement = wait.until(new ExpectedCondition<WebElement>() {

                                @NullableDecl

                                public WebElement apply(@NullableDecl WebDriver webDriver) {

                                    return webDriver.findElement(By.xpath("//*[@id=\"content\"]/div/div/table[1]"));

                                }

                            });

                        } catch (Exception ex) {

                            ex.printStackTrace();

                            continue;

                        }

                    }

                }

                List<WebElement> tds = contentElement.findElements(By.tagName("td"));

                try {

                    if (CollectionUtil.isNotEmpty(tds)){

                        YaoDianInfo dto = new YaoDianInfo();

                        dto.setCode(tds.get(2).getText());

                        dto.setBusinessRange(tds.get(18).getText());

                        dto.setBusinessStyle(tds.get(16).getText());

                        dto.setFazheng_date(tds.get(20).getText());

                        dto.setFazheng_jigou(tds.get(24).getText());

                        dto.setLeader(tds.get(12).getText());

                        dto.setLegalRepresentative(tds.get(10).getText());

                        dto.setName(tds.get(4).getText());

                        dto.setQualityLeader(tds.get(14).getText());

                        dto.setRegisterAddress(tds.get(6).getText());

                        dto.setWarehouse(tds.get(8).getText());

                        dto.setZhengshuyouxiaoqi_date(tds.get(22).getText());

                        dto.setXukezheng_status(tds.get(26).getText());

                        dto.setYdId(yaoDian.getId());

                        System.out.println(dto);

                        yaoDianInfoService.saveYaodianInfo(dto);

                    }

                } catch (Exception e) {

                    e.printStackTrace();

                    continue;

                }

            }

        } finally {

            driver.quit();

        }

    }

 

//    @Scheduled(fixedRate = 60*1000*30)

    public void grabYaoDian() throws Exception {

        Thread.sleep(3000);

        //总共40728页

        if (pageNum>40728){

            return;

        }

        //设置driver

        System.setProperty("webdriver.gecko.driver", "D:\\geckodriver/geckodriver.exe");

        //待整个dom树加载完成

        FirefoxOptions firefoxOptions = new FirefoxOptions();

        firefoxOptions.setPageLoadStrategy(PageLoadStrategy.EAGER);

        WebDriver driver = new FirefoxDriver(firefoxOptions);

        //到药品查询首页

        driver.get(HOME_URL);

        Thread.sleep(3000);

        int max=pageNum+300;

        try {

            for (int i=pageNum;i<=max;i++) {

                try {

                    Thread.sleep(1500);

                    List<YaoDian> yaoDianList = GrabYaoDianTask.executeByPage(driver, i);

                    try {

                        yaoDianService.saveBatch(yaoDianList);

                    } catch (Exception e) {

                        e.printStackTrace();

                        continue;

                    }

                } catch (Exception e) {

                    e.printStackTrace();

                    Thread.sleep(1000);

                    WebElement back = driver.findElement(By.xpath("//*[@id=\"tr1p11\"]/td/table/tbody/tr/td[3]"));

                    back.click();

                    Thread.sleep(1000);

                    i--;

                }

                pageNum=i;

            }

        } finally {

            driver.quit();

        }

    }

}

 

因为网址比较卡,页面经常会返回错误页,代码中写了很多重试的逻辑比较乱,最终数据爬取成功。

 

 

案例2:爬取某医保中心数据

这个网站比较老,使用的flash技术,数据传输格式是application/amf,爬取接口、解析HTML、selenium爬取页面的方式都不可取,可以使用blazeds来模拟与服务端请求获取数据。

 

使用Charles抓到的amf包

步骤:

  1. 导入flex maven坐标,blazeds是flash公司出的一个java客户端工具,一个类似httpclient的客户端,可以模拟发送amf协议的请求,并解析响应。

<!-- https://mvnrepository.com/artifact/org.apache.flex.blazeds/flex-messaging-core -->

        <dependency>

            <groupId>org.apache.flex.blazeds</groupId>

            <artifactId>flex-messaging-core</artifactId>

            <version>4.7.2</version>

        </dependency>

        <!-- https://mvnrepository.com/artifact/org.apache.flex.blazeds/flex-messaging-common -->

        <dependency>

            <groupId>org.apache.flex.blazeds</groupId>

            <artifactId>flex-messaging-common</artifactId>

            <version>4.7.2</version>

        </dependency>

  1. 代码编写

package com.ciics.selenium.example;

 

import cn.hutool.core.bean.BeanUtil;

import cn.hutool.core.bean.copier.CopyOptions;

import com.ciics.selenium.entity.YuLinYb;

import flex.messaging.io.ArrayCollection;

import flex.messaging.io.amf.ASObject;

import flex.messaging.io.amf.MessageBody;

import flex.messaging.io.amf.client.AMFConnection;

import flex.messaging.io.amf.translator.ASTranslator;

import flex.messaging.messages.RemotingMessage;

import org.springframework.beans.BeanUtils;

import org.zreport.crud.entity.RequestEntity;

 

import java.util.ArrayList;

import java.util.HashMap;

import java.util.Map;

import java.util.UUID;

 

/**

 * @author cxy

 * @date 2021/3/1 16:38

 */

public class FlashExample {

 

    public static void main(String[] args) throws Exception {

        String url = "****/messagebroker/amf";

        RemotingMessage remotingMessage = getRemotingMessage();

        AMFConnection connection = new AMFConnection();

        connection.addHttpRequestHeader("Content-Type", "application/x-amf");

        connection.connect(url);

        connection.addAmfHeader("DSId", UUID.randomUUID().toString().replaceAll("-", ""));

        // call 方法的 command 对应请求中的 Target

        Object call = connection.call(null, remotingMessage);

        // 解析返回值

        // flex.messaging.messages.AcknowledgeMessage

        flex.messaging.messages.AcknowledgeMessage message = (flex.messaging.messages.AcknowledgeMessage) call;

        ASObject body = (ASObject) message.getBody();

        ArrayCollection  datas = (ArrayCollection) body.get("datas");

        for (Object data:datas){

            System.out.println(data);

            ASObject obj= (ASObject) data;

        }

    }

 

    public static RemotingMessage getRemotingMessage() {

        // content 数组中的第一个对象

        RemotingMessage remotingMessage = new RemotingMessage();

        // 对应对象里面的各个参数

        remotingMessage.setSource(null);

        remotingMessage.setOperation("execute");

        remotingMessage.setDestination("ylzamfservice");

        remotingMessage.setTimeToLive(0);

        remotingMessage.setTimestamp(0);

        Map<String, Object> headers = new HashMap<String, Object>();

        headers.put("DSId", UUID.randomUUID().toString());

        headers.put("DSEndpoint", null);

        remotingMessage.setHeaders(headers);

        RequestEntity paramObj = new RequestEntity();

        paramObj.setServiceid("cp2query");

        paramObj.setLoginname("sddlsjcx");

        paramObj.setPassword("3706536a9be8e87f4e58ecf3d348047a");

        paramObj.setSessionid("NbHsssskG9iDBJCf6cVCDWHAIs0dPAPDCXtoxLfg8C0idm2saJFM!2051497475!1614583876388");

        HashMap<String, Object> map = new HashMap<String, Object>();

        map.put("selectSQL","select k.aae072,tr_aab301(k.aab301) as aaa103,to_char(k.aae036, 'yyyymmdd') as aae036,k.akc021,k.aac003,k.aac001,tr_aab004(k.aab001) aac010,k.aac004,k.akc023,k.aka120||decode(k.skc057,null,'','+'||k.skc057)||decode(k.skc058,null,'','+'||k.skc058) aka120,k.akc264,k.akc260,nvl(k.mztclj, 0) mztclj,nvl(k.aae005, '无') aae005,decode(k.bkc224, '000', '否', '001', '是', '否') as bkc224,decode(k.bkc225, '000', '否', '001', '是', '否') as bkc225,tr_akb020(k.akb020) as orgname,nvl(k.akc264, 0) - nvl(k.akc261, 0) - nvl(k.akc262, 0) -nvl(k.skc590,0) jgdfje,k.akc262, k.ake181 yljzjj,k.bkc220 dbbxzf,k.bkc121 dbjjzf,nvl(k.bkc222,0)-nvl(k.bkc228,0) fpjjzf  from kcao k where k.skc033 = 'Z' and k.akc021 like '9%' and substr(k.aab301, 0, 4) = '6108' and k.skc191 = '102' and (k.akb020 = '999' or '999'= '999') and k.aac001 =:aac001  and k.akc021 =:akc021 and k.bkc224 =:bkc224  and k.bkc225 =:bkc225 and k.sflb =:sflb and (k.aab301 = '999' or '999'= '999') and (k.aab034 = '999' or '999'= '999') and decode(k.ske801,null,'0','1') like '%' and   k.aae036 BETWEEN TO_DATE('20200101', 'YYYYMMDD') AND to_date('20200131235959','YYYYMMDDHH24MISS')");

        map.put("page",1);

        map.put("pagesize",100);

        map.put("rowcount",0);

        map.put("aae036_qs","20200101");

        map.put("aab301","999");

        paramObj.setParams(map);

        ArrayList<Map> list = new ArrayList<Map>();

        list.add(map);

        paramObj.setDatas(list);

        Object[] body = new Object[]{paramObj};

        remotingMessage.setBody(body);

        remotingMessage.setClientId(UUID.randomUUID().toString());

        remotingMessage.setMessageId(UUID.randomUUID().toString());

        return remotingMessage;

    }

}

 

Logo

魔乐社区(Modelers.cn) 是一个中立、公益的人工智能社区,提供人工智能工具、模型、数据的托管、展示与应用协同服务,为人工智能开发及爱好者搭建开放的学习交流平台。社区通过理事会方式运作,由全产业链共同建设、共同运营、共同享有,推动国产AI生态繁荣发展。

更多推荐