手把手教你Python抓取数据并可视化_python_

import pprint pprint.pprint(parse_data_dict) 

import requests import re import json import csv import time from random import random from fake_useragent import UserAgent def spider_python(key_word): headers = { 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9', 'Accept-Language': 'zh-CN,zh;q=0.9', 'Cache-Control': 'no-cache', 'Connection': 'keep-alive', 'Pragma': 'no-cache', 'Sec-Fetch-Dest': 'document', 'Sec-Fetch-Mode': 'navigate', 'Sec-Fetch-Site': 'same-origin', 'Sec-Fetch-User': '?1', 'Upgrade-Insecure-Requests': '1', 'User-Agent': UserAgent().Chrome, 'sec-ch-ua': '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"', 'sec-ch-ua-mobile': '?0', 'sec-ch-ua-platform': '"Windows"', } params = { 'lang': 'c', 'postchannel': '0000', 'workyear': '99', 'cotype': '99', 'degreefrom': '99', 'jobterm': '99', 'companysize': '99', 'ord_field': '0', 'dibiaoid': '0', 'line': '', 'welfare': '', } save_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()).replace(' ', '_').replace(':','_') file_path = f'./testDataPython-{save_time}.csv' f_csv = open(file_path, mode='w', encoding='utf-8', newline='') fieldnames = ['公司名字', '职位名字', '薪资', '工作地点', '招聘要求', '公司待遇','招聘更新时间', '招聘发布时间', '公司人数', '公司类型', 'companyind_text', 'job_href', 'company_href'] dict_write = csv.DictWriter(f_csv, fieldnames=fieldnames) dict_write.writeheader() page = 0 #页数 error_time = 0 #在判断 职位名字中是否没有关键字的次数，这里定义出现200次时，while循环结束 # （因为在搜索岗位名字时（如：搜索python）,会在网站20多页时就没有关于python的岗位了，但是仍然有其它的岗位出现，所以这里就需要if判断，使其while循环结束） flag = True while flag: page += 1 print(f'第{page}抓取中……') try: time.sleep(random()*3) #这里随机休眠一下，简单反爬处理，反正我们用的是单线程爬取，也不差那一点时间是吧 url='这里你们自己构建url吧，从上面的图片应该能看出，我写出来的话实在是不行，过不了审核，难受！！！' ###这里还是要添加cookies的好，我们要伪装好不是？防止反爬，如果你用上面提供的方法，也就很快的构建出cookies。 response = requests.get(url=url,params=params, headers=headers) except: print(f'\033[31m第{page}请求异常！033[0m') flag = False parse_data = re.findall('"engine_jds":(.*?),"jobid_count"',response.text) parse_data_dict = json.loads(parse_data[0]) # import pprint # pprint.pprint(parse_data_dict) # exit() for i in parse_data_dict: ###在这里要处理下异常，因为在爬取多页时，可能是网站某些原因会导致这里的结构变化 try: companyind_text = i['companyind_text'] except Exception as e: print(f'\033[31m异常：{e}033[0m') companyind_text = None dic = { '公司名字': i['company_name'], '职位名字': i['job_name'], '薪资': i['providesalary_text'], '工作地点': i['workarea_text'], '招聘要求': ' '.join(i['attribute_text']), '公司待遇': i['jobwelf'], '招聘更新时间': i['updatedate'], '招聘发布时间': i['issuedate'], '公司人数': i['companysize_text'], '公司类型': i['companytype_text'], 'companyind_text': companyind_text, 'job_href': i['job_href'], 'company_href': i['company_href'], } if 'Python' in dic['职位名字'] or 'python' in dic['职位名字']: dict_write.writerow(dic) print(dic['职位名字'], '——保存完毕！') else: error_time += 1 if error_time == 200: flag = False print('抓取完成！') f_csv.close() if __name__ == '__main__': key_word = 'python' # key_word = 'java' ##这里不能输入中文，网址做了url字体加密，简单的方法就是直接从网页url里面复制下来用（如：前端） # key_word = '%25E5%2589%258D%25E7%25AB%25AF' #前端 spider_python(key_word) 

import pandas as pd from pyecharts import options as opts python_data = pd.read_csv('./testDataPython-2022-05-01_11_48_36.csv') python_data['工作地点'] = [i.split('-')[0] for i in python_data['工作地点']] city = python_data['工作地点'].value_counts() ###柱状图 from pyecharts.charts import Bar c = ( Bar() .add_xaxis(city.index.tolist()) #城市列表数据项 .add_yaxis("Python", city.values.tolist())#城市对应的岗位数量列表数据项 .set_global_opts( title_opts=opts.TitleOpts(title="Python招聘岗位所在城市分布情况"), datazoom_opts=[opts.DataZoomOpts(), opts.DataZoomOpts(type_="inside")], xaxis_opts=opts.AxisOpts(name='城市'), # 设置x轴名字属性 yaxis_opts=opts.AxisOpts(name='岗位数量'), # 设置y轴名字属性 ) .render("bar_datazoom_both.html") ) 

import pandas as pd import copy from pyecharts import options as opts python_data = pd.read_csv('./testDataPython-2022-05-01_11_48_36.csv') python_data_deepcopy = copy.deepcopy(python_data) #深复制一份数据 python_data['工作地点'] = [i.split('-')[0] for i in python_data['工作地点']] city = python_data['工作地点'].value_counts() city_list = [list(ct) for ct in city.items()] def province_city(): '''这是从接口里爬取的数据（不太准，但是误差也可以忽略不计！）''' area_data = {} with open('./中国省份_城市.txt', mode='r', encoding='utf-8') as f: for line in f: line = line.strip().split('_') area_data[line[0]] = line[1].split(',') province_data = [] for ct in city_list: for k, v in area_data.items(): for i in v: if ct[0] in i: ct[0] = k province_data.append(ct) area_data_deepcopy = copy.deepcopy(area_data) for k in area_data_deepcopy.keys(): area_data_deepcopy[k] = 0 for i in province_data: if i[0] in area_data_deepcopy.keys(): area_data_deepcopy[i[0]] = area_data_deepcopy[i[0]] +i[1] province_data = [[k,v]for k,v in area_data_deepcopy.items()] best = max(area_data_deepcopy.values()) return province_data,best province_data,best = province_city() #地图_中国地图（带省份）Map-VisualMap（连续型） c2 = ( Map() .add( "Python",province_data, "china") .set_global_opts( title_opts=opts.TitleOpts(title="Python招聘岗位——全国分布情况"), visualmap_opts=opts.VisualMapOpts(max_=int(best / 2)), ) .render("map_china.html") ) 

import requests import json header = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36", } response = requests.get('https://j.i8tq.com/weather2020/search/city.js',headers=header) result = json.loads(response.text[len('var city_data ='):]) print(result) each_province_data = {} f = open('./中国省份_城市.txt',mode='w',encoding='utf-8') for k,v in result.items(): province = k if k in ['上海', '北京', '天津', '重庆']: city = '，'.join(list(v[k].keys())) else: city = '，'.join(list(v.keys())) f.write(f'{province}_{city}\n') each_province_data[province] = city f.close() print(each_province_data)