效果展示

为保证画质，建议在Wi-Fi环境下观看

实现功能

• 获取天天基金网中一万多支基金的基金代码，基金名称

• 根据得到的基金代码，爬取自基金成立以来至今天的所有历史净值，日增长率等近百万条信息，保存进Mysql数据库中

• 编写Spider类，用于解析response数据，然后继续跟进url，并交给调度器请求，对跟进之后返回的response数据进行提取，得到想要的数据

# -*- coding: utf-8 -*-import scrapyimport reimport jsonimport Get_User_Agentimport timeclass SpiderFundSpider(scrapy.Spider):    name = 'Spider_Fund'    allowed_domains = ['fund.eastmoney.com']    start_urls = ['https://fund.eastmoney.com']    def __init__(self):        self.num = 0        self.All_dict = {}    # 爬虫开始，执行的方法    def start_requests(self):        '''        最开始时执行，不受域名的限制        '''        # 构建request对象，并交由parse_start回调方法处理        request = scrapy.FormRequest(            url='http://fund.eastmoney.com/allfund.html',            callback=self.parse        )        print('正在爬取基金代码与基金名称...')        yield request    def parse(self, response):        fund_list = response.xpath("//ul[@class = 'num_right']//li//a[1]/text()").extract()        code_name = []        for i in fund_list:            ls_list = []            ls_list.append(str(i).split(')')[0].replace(' ','').replace('(',''))            ls_list.append(str(i).split(')')[1].replace(' ', ''))            self.All_dict[ls_list[0]] = ls_list[1]            code_name.append(ls_list)        print('基金代码与基金名称爬取完成！')        # 循环打印输出        for i in range(len(code_name)):            print('第' + str(i + 1) + '/' + str(len(code_name)) + '只基金\t' + str(code_name[i][0]) + str(code_name[i][1]))        # 循环打印输出        for i in range(len(code_name)):            self.num = 0            # 只爬取100条            for page in range(1, 2):                params = {                    'callback': 'jQuery18303760298133681277_1608898296152',                    'fundCode': str(code_name[i][0]),                    'pageIndex': page,                    'pageSize': 100,                }                # 构建request对象，并交由parse_page1回调方法处理                request = scrapy.FormRequest(                    url='http://api.fund.eastmoney.com/f10/lsjz?callback=%s&fundCode=%s&pageIndex=%s&pageSize=%s' % (str(params['callback']),str(params['fundCode']),str(params['pageIndex']),str(params['pageSize'])),                    headers={                        'Cookie': 'EMFUND1=null; EMFUND2=null; EMFUND3=null; EMFUND4=null; AUTH_FUND.EASTMONEY.COM_GSJZ=AUTH*TTJJ*TOKEN; qgqp_b_id=2f525ea6e3ec44bf1596c1a046d499ae; cowCookie=true; intellpositionL=1079.19px; intellpositionT=455px; waptgshowtime=20201224; _qddaz=QD.fuh01.gyu07o.kj2syf2w; pgv_pvi=1820343296; st_si=11009327950870; st_asi=delete; EMFUND0=null; EMFUND5=12-24%2019%3A37%3A01@%23%24%u62DB%u5546%u4E2D%u8BC1%u767D%u9152%u6307%u6570%u5206%u7EA7@%23%24161725; EMFUND6=12-23%2021%3A10%3A59@%23%24%u6613%u65B9%u8FBE%u6D88%u8D39%u884C%u4E1A%u80A1%u7968@%23%24110022; EMFUND7=12-23%2021%3A12%3A47@%23%24%u5174%u534E%u6C38%u5174%u6DF7%u5408%u53D1%u8D77%u5F0FC@%23%24010757; EMFUND8=12-24%2021%3A47%3A08@%23%24%u878D%u901A%u901A%u4F18%u503A%u5238@%23%24003146; EMFUND9=12-25 20:10:58@#$%u4FE1%u8BDA%u65B0%u5174%u4EA7%u4E1A%u6DF7%u5408@%23%24000209; st_pvi=41441525871173; st_sp=2020-12-20%2011%3A45%3A31; st_inirUrl=http%3A%2F%2Ffund.eastmoney.com%2Fdata%2Ffundranking.html; st_sn=6; st_psi=2020122520113686-0-1647922279',                        'Host': 'api.fund.eastmoney.com',                        'User-Agent': Get_User_Agent.Get_User_Agent(),                        'Referer': 'http://fundf10.eastmoney.com/jjjz_%s.html' % str(code_name[i][0]),                    },                    # 回调函数                    callback=self.parse_next                )                # 保存在request的meta中   后续会使用到                namelist = list(code_name[i])                request.meta['name'] = namelist                yield request    def parse_next(self, response):        data_dict = {}        if response.status == 200:            text = re.findall('\((.*?)\)', response.text)[0]  # 提取dict            data_tuple = (response.meta['name'][0],response.meta['name'][1])            try:                data_dict[data_tuple] = json.loads(text)['Data']['LSJZList']            except:                print('异常了' + response.url)            print('已爬取\t%s%s\t%s\t条净值信息' % (str(response.url).split('&')[1].split('=')[1],                                                 self.All_dict[str(response.url).split('&')[1].split('=')[1]],                                                 str(int(str(response.url).split('&')[2].split('=')[1])*100)))        else:            print(response.url)        yield data_dict

各只基金的基金代码，基金名称

自基金成立以来至今天

的所有历史净值，日增长率

• 编写管道Pipelines文件，处理spider中获取的数据，并对数据进行后期处理(过滤，存储等)

import pymysqlimport jsonclass FundPipeline(object):    def __init__(self):        '''        连接数据库        '''        self.mysql_conn = pymysql.Connection(            host='localhost',            port=3306,            user='test',            password='test',            database='cll',            charset='utf8',        )        self.num = 0    def process_item(self, item, spider):        sql_column = "FUND_CODE,FUND_NAME,FSRQ,DWJZ,LJJZ,JZZZL,SGZT,SHZT"        sql_value = ""        for key,value in item.items():            for fund_dict in value:                sql_value = sql_value + "('%s','%s','%s','%s','%s','%s','%s','%s')," % (key[0],key[1],fund_dict['FSRQ'],fund_dict['DWJZ'],fund_dict['LJJZ'],fund_dict['JZZZL'],fund_dict['SGZT'],fund_dict['SHZT'])                self.num = self.num + 1        # 创建光标对象        cs = self.mysql_conn.cursor()        if sql_value:            # 拼接sql语句            sql_str = 'insert into fund (%s) values %s;' % (sql_column, sql_value.rstrip(','))            try:                # 执行SQL语句                cs.execute(sql_str)                # 提交                self.mysql_conn.commit()            except:                print('哎,SQL异常了,瞅一眼呗')                print(sql_str)

• setting配置文件

BOT_NAME = 'Fund'SPIDER_MODULES = ['Fund.spiders']NEWSPIDER_MODULE = 'Fund.spiders'# Obey robots.txt rules   # 不遵守Robot协议ROBOTSTXT_OBEY = False# Configure maximum concurrent requests performed by Scrapy (default: 16)# 线程数CONCURRENT_REQUESTS = 100# 下载延时DOWNLOAD_DELAY = 3# 开启Spider中间件# SPIDER_MIDDLEWARES = {#    'Fund.middlewares.FundSpiderMiddleware': 543,# }# Downloader中间件DOWNLOADER_MIDDLEWARES = {   'Fund.middlewares.FundDownloaderMiddleware': 300,}# Configure item pipelines# See https://docs.scrapy.org/en/latest/topics/item-pipeline.htmlITEM_PIPELINES = {   'Fund.pipelines.FundPipeline': 300,}

• 项目启动文件

from scrapy.cmdline import executeexecute('scrapy crawl Spider_Fund'.split())

最终效果

为保证画质，建议在Wi-Fi环境下观看