毕业设计 ---- 基于大数据挖掘的职位薪资水平分析--数据预处理

爬虫爬取数据

数据来源于boss直聘网，通过爬虫采集
采集的城市主要为一线、新一线等较为发达的城市
爬虫代码如下：

from selenium import webdriver
from bs4 import BeautifulSoup

driver = webdriver.Chrome(r'D:\PycharmProjects\python_present\boss直聘爬取\chromedriver.exe')

cities = [{"name": "北京", "code": 101010100, "url": "/beijing/"},
{"name": "上海", "code": 101020100, "url": "/shanghai/"},
{"name": "广州", "code": 101280100, "url": "/guangzhou/"},
{"name": "深圳", "code": 101280600, "url": "/shenzhen/"},
{"name": "杭州", "code": 101210100, "url": "/hangzhou/"},
{"name": "天津", "code": 101030100, "url": "/tianjin/"},
{"name": "苏州", "code": 101190400, "url": "/suzhou/"},
{"name": "武汉", "code": 101200100, "url": "/wuhan/"},
{"name": "厦门", "code": 101230200, "url": "/xiamen/"},
{"name": "长沙", "code": 101250100, "url": "/changsha/"},
{"name": "成都", "code": 101270100, "url": "/chengdu/"},
{"name": "郑州", "code": 101180100, "url": "/zhengzhou/"},
{"name": "重庆", "code": 101040100, "url": "/chongqing/"},
{"name": "青岛", "code": 101120200, "url": "/qingdao/"},
{"name": "南京", "code": 101190100, "url": "/nanjing/"}]

for city in cities:
urls = ['https://www.zhipin.com/c{}/?query=数据分析&page={}&ka=page-{}'.format(city['code'], i, i) for i in
range(1, 8)]
for url in urls:
driver.get(url)
html = driver.page_source
bs = BeautifulSoup(html, 'html.parser')
job_all = bs.find_all('div', {"class": "job-primary"})
for job in job_all:
position = job.find('span', {"class": "job-name"}).get_text()
address = job.find('span', {'class': "job-area"}).get_text()
company = job.find('div', {'class': 'company-text'}).find('h3', {'class': "name"}).get_text()
salary = job.find('span', {'class': 'red'}).get_text()
diploma = job.find('div', {'class': 'job-limit'}).find('p').get_text()[-2:]
experience = job.find('div', {'class': 'job-limit'}).find('p').get_text()[:-2]
labels = job.find('a', {'class': 'false-link'}).get_text()
with open('position.csv', 'a+', encoding='UTF-8-SIG') as f_obj:
f_obj.write(position.replace(',', '、') + "," + address + "," + company + "," + salary + "," + diploma+ "," + experience + ',' + labels + "\n")    


driver.quit()

数据清洗

cols=list(df)
cols.insert(4,cols.pop(cols.index('bottom')))
cols.insert(5,cols.pop(cols.index('top')))
cols.insert(6,cols.pop(cols.index('commision_pct')))
cols.insert(7,cols.pop(cols.index('avg_salary')))
df=df.loc[:,cols]