1、.net core 3.1 C#  selenium 

//爬取所有省份、城市、区县
public override async Task WriteAreaToFileAsync(string configPath, string directory)
        {
            const string BaseProvinceLink = "https://www.qcc.com/search?key={keyword}#industrycode:K&";
            const string BaseCityLink = "https://www.qcc.com/search_getCityListHtml?province={0}";
            const string BaseCountyLink = "https://www.qcc.com/search_getCountyListHtml?city={0}";
            List<string> provinces = await GetCodeAsync(new Uri(BaseProvinceLink), ".sfilter-tag.clearfix.provinceChoose dd a");
            const string baseText = "province:{0}&city:{1}&county:{2}&";
            List<string> list = new List<string>();
            foreach (var province in provinces)
            {
                StringBuilder.Clear();
                Uri provinceUri = new Uri(StringBuilder.AppendFormat(BaseCityLink, province).ToString());
                var cities = await GetCodeAsync(provinceUri, "dd a");
                foreach (var city in cities)
                {
                    StringBuilder.Clear();
                    Uri cityUri = new Uri(StringBuilder.AppendFormat(BaseCountyLink, city).ToString());
                    var counties = await GetCodeAsync(cityUri, "dd a");
                    foreach (var county in counties)
                    {
                        StringBuilder.Clear();
                        StringBuilder.Append(BaseProvinceLink);
                        string area = StringBuilder.AppendFormat(baseText, province, city, county).
                            Replace("search", "search_index").Replace("中介#", "中介&ajaxflag=1&")
                            .Replace(":industrycode", "=industrycode").ToString();
                        list.Add(area);
                    }
                }
            }
            await File.WriteAllLinesAsync("企查查.txt", list);
        }

//分页爬取企业信息
        private async Task<bool> GetAgentsAsync(Uri cityUri)
        {
            LogHelper.Info(cityUri.ToString());
            var pageSource = await HttpClient.GetStringAsync(cityUri);
            while (!pageSource.Contains("查企业"))
            {
                if (pageSource.StartsWith("<script>window.location"))
                {
                    VertifyCode(new Uri(pageSource.Split("'")[1]));
                    pageSource = await HttpClient.GetStringAsync(cityUri);
                }
                else if (pageSource.Contains("小查还没找到数据"))
                {
                    return false;
                }
            }
            var block = JumonyParser.Parse(pageSource).Find(".m_srchList tbody tr td:nth-child(3)");
            foreach (var item in block)
            {
                await VertifyAsync(item.InnerHtml());
            }
            if (block.Count() < PageSize)
            {
                return false;
            }
            return true;
        }

2、结果截图

3、需要开通vip账号

4、过滑动验证码

Logo

魔乐社区(Modelers.cn) 是一个中立、公益的人工智能社区,提供人工智能工具、模型、数据的托管、展示与应用协同服务,为人工智能开发及爱好者搭建开放的学习交流平台。社区通过理事会方式运作,由全产业链共同建设、共同运营、共同享有,推动国产AI生态繁荣发展。

更多推荐