爬行和分析猫眼对电影《流浪地球》的评论

 1 import pandas as pd
 2 import numpy as np
 3 import matplotlib.pyplot as plt
 4 from pymongo import MongoClient
 5 from pandas.io.json import json_normalize
 6 %matplotlib inline
 7 
 8 conn = MongoClient(host='127.0.0.1', port=27017)  # 实例化MongoClient
 9 db = conn.get_database('maoyan')  # 连接到maoyan数据库
10 maoyan = db.get_collection('maoyan') # 连接到集合maoyan
11 mon_data = maoyan.find()  # 查询这个集合下的所有记录
12 
13 data = json_normalize([comment for comment in mon_data])
14 
15 data.info()

data.head()

1 # 数据清洗
2 data = data.drop(columns='_id')
3 data = data.drop_duplicates(subset='userId')
4 data['time'] = pd.to_datetime(data['time']/1000, unit='s')
5 data = data[data['time']>=pd.to_datetime('2019-02-05 00:00:00')]
6 data.set_index(data["time"], inplace=True)
7 data.head()

1 data.info()

data['score'].mean()

 1 from pyecharts import Bar
 2 from pyecharts import Line
 3 from pyecharts import Overlap
 4 
 5 score_total = data['score'].value_counts().sort_index()
 6 bar = Bar("《流浪地球》各评分数量", width=700)
 7 line = Line("", width=700)
 8 bar.add("", score_total.index, score_total.values, is_stack=True, is_label_show=True,
 9        bar_category_gap='40%', label_color = ['#130f40'],
10        legend_text_size=18,xaxis_label_textsize=18,yaxis_label_textsize=18)
11 line.add("", score_total.index, score_total.values+1000, is_smooth=True)
12 
13 overlap = Overlap(width=700)
14 overlap.add(bar)
15 overlap.add(line)
16 
17 overlap

1 # 低分百分比
2 score_total[:5].sum()/score_total.sum()*100

# 高分百分比
score_total[7:].sum()/score_total.sum()*100

# 满分百分比
score_total[10:].sum()/score_total.sum()*100

1 score_by_time = data['score'].resample('H').mean()
2 line = Line("《流浪地球》平均评分时间走向", width=700)
3 line.add("", score_by_time.index.date, score_by_time.values, is_smooth=True,
4          legend_text_size=18,xaxis_label_textsize=18,yaxis_label_textsize=18,
5          xaxis_rotate=20, yaxis_min=8)
6 line

score_by_time.nsmallest(6)

 1 import jieba
 2 from collections import Counter
 3 from pyecharts import WordCloud
 4 
 5 jieba.add_word('屈楚萧')
 6 jieba.add_word('刘启')
 7 jieba.add_word('吴京')
 8 jieba.add_word('刘培强')
 9 jieba.add_word('李光洁')
10 jieba.add_word('王磊')
11 jieba.add_word('吴孟达')
12 jieba.add_word('达叔')
13 jieba.add_word('韩子昂')
14 jieba.add_word('赵今麦')
15 jieba.add_word('韩朵朵')
16 
17 swords = [x.strip() for x in open ('stopwords.txt')]

 1 def plot_word_cloud(data, swords):
 2     text = ''.join(data['content'])
 3     words = list(jieba.cut(text))
 4     ex_sw_words = []
 5     for word in words:
 6         if len(word)>1 and (word not in swords):
 7             ex_sw_words.append(word)
 8     c = Counter()
 9     c = Counter(ex_sw_words)
10     wc_data = pd.DataFrame({'word':list(c.keys()), 'counts':list(c.values())}).sort_values(by='counts', ascending=False).head(100)
11     wordcloud = WordCloud(width=1300, height=620)
12     wordcloud.add("", wc_data['word'], wc_data['counts'], word_size_range=[20, 100])
13     return wordcloud
14 # 高分的评价
15 plot_word_cloud(data=data[data['score']>6], swords=swords)

1 data[data['score']>6].nlargest(10, 'upCount')

1 for i in data[data['score']>6].nlargest(10, 'upCount')['content']:
2     print(i+'\n')

1 # 低分的评价
2 plot_word_cloud(data=data[data['score']<5], swords=swords)

1 data[data['score']<5].nlargest(10, 'upCount')

1 for i in data[data['score']<5].nlargest(10, 'upCount')['content']:
2     print(i+'\n')

1 # 总体的性别比例
2 gender_total = data['gender'].value_counts()
3 bar = Bar("《流浪地球》观众性别", width=700)
4 bar.add("", ['未知', '男', '女'], gender_total.values, is_stack=True, is_label_show=True,
5        bar_category_gap='60%', label_color = ['#130f40'],
6        legend_text_size=18,xaxis_label_textsize=18,yaxis_label_textsize=18)
7 bar
8 gender_total/gender_total.sum()*100

 1 # 低分的性别比例
 2 
 3 gender_low = data.loc[data['score']<5, 'gender'].value_counts()
 4 
 5 bar = Bar("《流浪地球》低分评论观众性别", width=700)
 6 
 7 bar.add("", ['未知'zh, '男', '女'], gender_low.values, is_stack=True, is_label_show=True, bar_category_gap='60%', label_color = ['#130f40'],
 8 
 9 legend_text_size=18,xaxis_label_textsize=18,yaxis_label_textsize=18)
10 
11 bar
12 
13 gender_low/gender_low.sum()*100