一、环境准备
- python3.8.3
- pycharm
- 项目所需第三方包
|
1
|
pip install scrapy fake-useragent requests selenium virtualenv -i https://pypi.douban.com/simple |
1.1 创建虚拟环境
切换到指定目录创建
|
1
|
virtualenv .venv |
创建完记得激活虚拟环境
1.2 创建项目
|
1
|
scrapy startproject 项目名称 |
1.3 使用pycharm打开项目，将创建的虚拟环境配置到项目中来
1.4 创建京东spider
|
1
|
scrapy genspider 爬虫名称 url |
1.5 修改允许访问的域名，删除https：
二、问题分析
爬取数据的思路是先获取首页的基本信息，在获取详情页商品详细信息；爬取京东数据时，只返回40条数据，这里，作者使用selenium，在scrapy框架中编写下载器中间件，返回页面所有数据。
爬取的字段分别是：
商品价格
商品评数
商品店家
商品sku（京东可直接搜索到对应的产品）
商品标题
商品详细信息
三、spider
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
|
import reimport scrapyfrom lianjia.items import jd_detailitemclass jicomputerdetailspider(scrapy.spider): name = 'ji_computer_detail' allowed_domains = ['search.jd.com', 'item.jd.com'] start_urls = [ 'https://search.jd.com/search?keyword=%e7%ac%94%e8%ae%b0%e6%9c%ac%e7%94%b5%e8%84%91&suggest=1.def.0.base&wq=%e7%ac%94%e8%ae%b0%e6%9c%ac%e7%94%b5%e8%84%91&page=1&s=1&click=0'] def parse(self, response): lls = response.xpath('//ul[@class="gl-warp clearfix"]/li') for ll in lls: item = jd_detailitem() computer_price = ll.xpath('.//div[@class="p-price"]/strong/i/text()').extract_first() computer_commit = ll.xpath('.//div[@class="p-commit"]/strong/a/text()').extract_first() computer_p_shop = ll.xpath('.//div[@class="p-shop"]/span/a/text()').extract_first() item['computer_price'] = computer_price item['computer_commit'] = computer_commit item['computer_p_shop'] = computer_p_shop meta = { 'item': item } shop_detail_url = ll.xpath('.//div[@class="p-img"]/a/@href').extract_first() shop_detail_url = 'https:' + shop_detail_url yield scrapy.request(url=shop_detail_url, callback=self.detail_parse, meta=meta) for i in range(2, 200, 2): next_page_url = f'https://search.jd.com/search?keyword=%e7%ac%94%e8%ae%b0%e6%9c%ac%e7%94%b5%e8%84%91&suggest=1.def.0.base&wq=%e7%ac%94%e8%ae%b0%e6%9c%ac%e7%94%b5%e8%84%91&page={i}&s=116&click=0' yield scrapy.request(url=next_page_url, callback=self.parse) def detail_parse(self, response): item = response.meta.get('item') computer_sku = response.xpath('//a[@class="notice j-notify-sale"]/@data-sku').extract_first() item['computer_sku'] = computer_sku computer_title = response.xpath('//div[@class="sku-name"]/text()').extract_first().strip() computer_title = ''.join(re.findall('\s', computer_title)) item['computer_title'] = computer_title computer_detail = response.xpath('string(//ul[@class="parameter2 p-parameter-list"])').extract_first().strip() computer_detail = ''.join(re.findall('\s', computer_detail)) item['computer_detail'] = computer_detail yield item |
四、item
|
1
2
3
4
5
6
7
8
|
class jd_detailitem(scrapy.item): # define the fields for your item here like: computer_sku = scrapy.field() computer_price = scrapy.field() computer_title = scrapy.field() computer_commit = scrapy.field() computer_p_shop = scrapy.field() computer_detail = scrapy.field() |
五、setting
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
|
import randomfrom fake_useragent import useragentua = useragent()user_agent = ua.randomrobotstxt_obey = falsedownload_delay = random.uniform(0.5, 1)downloader_middlewares = { 'lianjia.middlewares.jddownloadermiddleware': 543}item_pipelines = { 'lianjia.pipelines.jd_csv_pipeline': 300} |
六、pipelines
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
|
class jd_csv_pipeline: # def process_item(self, item, spider): # return item def open_spider(self, spider): self.fp = open('./jd_computer_message.xlsx', mode='w+', encoding='utf-8') self.fp.write('computer_sku\tcomputer_title\tcomputer_p_shop\tcomputer_price\tcomputer_commit\tcomputer_detail\n') def process_item(self, item, spider): # 写入文件 try: line = '\t'.join(list(item.values())) + '\n' self.fp.write(line) return item except: pass def close_spider(self, spider): # 关闭文件 self.fp.close() |
七、middlewares
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
|
class jddownloadermiddleware: def process_request(self, request, spider): # 判断是否是ji_computer_detail的爬虫 # 判断是否是首页 if spider.name == 'ji_computer_detail' and re.findall(f'.*(item.jd.com).*', request.url) == []: options = chromeoptions() options.add_argument("--headless") driver = webdriver.chrome(options=options) driver.get(request.url) for i in range(0, 15000, 5000): driver.execute_script(f'window.scrollto(0, {i})') time.sleep(0.5) body = driver.page_source.encode() time.sleep(1) return htmlresponse(url=request.url, body=body, request=request) return none |
八、使用jupyter进行简单的处理和分析
其他文件：百度停用词库、简体字文件
下载第三方包
|
1
|
!pip install seaborn jieba wordcloud pil -i https://pypi.douban.com/simple |
8.1导入第三方包
|
1
2
3
4
5
6
7
8
9
10
11
12
|
import reimport osimport jiebaimport wordcloudimport pandas as pdimport numpy as npfrom pil import imageimport seaborn as snsfrom docx import documentfrom docx.shared import inchesimport matplotlib.pyplot as pltfrom pandas import dataframe,series |
8.2设置可视化的默认字体和seaborn的样式
|
1
2
3
|
sns.set_style('darkgrid')plt.rcparams['font.sans-serif'] = ['simhei']plt.rcparams['axes.unicode_minus'] = false |
8.3读取数据
|
1
|
df_jp = pd.read_excel('./jd_shop.xlsx') |
8.4筛选inteli5、i7、i9处理器数据
|
1
2
3
4
5
6
7
8
|
def convert_one(s): if re.findall(f'.*?(i5).*', str(s)) != []: return re.findall(f'.*?(i5).*', str(s))[0] elif re.findall(f'.*?(i7).*', str(s)) != []: return re.findall(f'.*?(i7).*', str(s))[0] elif re.findall(f'.*?(i9).*', str(s)) != []: return re.findall(f'.*?(i9).*', str(s))[0]df_jp['computer_intel'] = df_jp['computer_detail'].map(convert_one) |
8.5筛选笔记本电脑的屏幕尺寸范围
|
1
2
3
4
|
def convert_two(s): if re.findall(f'.*?(\d+\.\d+英寸-\d+\.\d+英寸).*', str(s)) != []: return re.findall(f'.*?(\d+\.\d+英寸-\d+\.\d+英寸).*', str(s))[0]df_jp['computer_in'] = df_jp['computer_detail'].map(convert_two) |
8.6将评论数转化为整形
|
1
2
3
4
5
6
7
8
|
def convert_three(s): if re.findall(f'(\d+)万+', str(s)) != []: number = int(re.findall(f'(\d+)万+', str(s))[0]) * 10000 return number elif re.findall(f'(\d+)+', str(s)) != []: number = re.findall(f'(\d+)+', str(s))[0] return numberdf_jp['computer_commit'] = df_jp['computer_commit'].map(convert_three) |
8.7筛选出需要分析的品牌
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
|
def find_computer(name, s): sr = re.findall(f'.*({name}).*', str(s))[0] return srdef convert(s): if re.findall(f'.*(联想).*', str(s)) != []: return find_computer('联想', s) elif re.findall(f'.*(惠普).*', str(s)) != []: return find_computer('惠普', s) elif re.findall(f'.*(华为).*', str(s)) != []: return find_computer('华为', s) elif re.findall(f'.*(戴尔).*', str(s)) != []: return find_computer('戴尔', s) elif re.findall(f'.*(华硕).*', str(s)) != []: return find_computer('华硕', s) elif re.findall(f'.*(小米).*', str(s)) != []: return find_computer('小米', s) elif re.findall(f'.*(荣耀).*', str(s)) != []: return find_computer('荣耀', s) elif re.findall(f'.*(神舟).*', str(s)) != []: return find_computer('神舟', s) elif re.findall(f'.*(外星人).*', str(s)) != []: return find_computer('外星人', s)df_jp['computer_p_shop'] = df_jp['computer_p_shop'].map(convert) |
8.8删除指定字段为空值的数据
|
1
2
3
|
for n in ['computer_price', 'computer_commit', 'computer_p_shop', 'computer_sku', 'computer_detail', 'computer_intel', 'computer_in']: index_ls = df_jp[df_jp[[n]].isnull().any(axis=1)==true].index df_jp.drop(index=index_ls, inplace=true) |
8.9查看各品牌的平均价格
|
1
2
3
4
5
6
7
8
9
|
plt.figure(figsize=(10, 8), dpi=100)ax = sns.barplot(x='computer_p_shop', y='computer_price', data=df_jp.groupby(by='computer_p_shop')[['computer_price']].mean().reset_index())for index,row in df_jp.groupby(by='computer_p_shop')[['computer_price']].mean().reset_index().iterrows(): ax.text(row.name,row['computer_price'] + 2,round(row['computer_price'],2),color="black",ha="center")ax.set_xlabel('品牌')ax.set_ylabel('平均价格')ax.set_title('各品牌平均价格')boxplot_fig = ax.get_figure()boxplot_fig.savefig('各品牌平均价格.png', dpi=400) |
8.10 查看各品牌的价格区间
|
1
2
3
4
5
6
7
|
plt.figure(figsize=(10, 8), dpi=100)ax = sns.boxenplot(x='computer_p_shop', y='computer_price', data=df_jp.query('computer_price>500'))ax.set_xlabel('品牌')ax.set_ylabel('价格区间')ax.set_title('各品牌价格区间')boxplot_fig = ax.get_figure()boxplot_fig.savefig('各品牌价格区间.png', dpi=400) |
8.11 查看价格与评论数的关系
|
1
2
3
|
df_jp['computer_commit'] = df_jp['computer_commit'].astype('int64')ax = sns.jointplot(x="computer_commit", y="computer_price", data=df_jp, kind="reg", truncate=false,color="m", height=10)ax.fig.savefig('评论数与价格的关系.png') |
8.12 查看商品标题里出现的关键词
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
|
import imageio# 将特征转换为列表ls = df_jp['computer_title'].to_list()# 替换非中英文的字符feature_points = [re.sub(r'[^a-za-z\u4e00-\u9fa5]+',' ',str(feature)) for feature in ls]# 读取停用词stop_world = list(pd.read_csv('./百度停用词表.txt', engine='python', encoding='utf-8', names=['stopwords'])['stopwords'])feature_points2 = []for feature in feature_points: # 遍历每一条评论 words = jieba.lcut(feature) # 精确模式，没有冗余.对每一条评论进行jieba分词 ind1 = np.array([len(word) > 1 for word in words]) # 判断每个分词的长度是否大于1 ser1 = pd.series(words) ser2 = ser1[ind1] # 筛选分词长度大于1的分词留下 ind2 = ~ser2.isin(stop_world) # 注意取反负号 ser3 = ser2[ind2].unique() # 筛选出不在停用词表的分词留下,并去重 if len(ser3) > 0: feature_points2.append(list(ser3))# 将所有分词存储到一个列表中wordlist = [word for feature in feature_points2 for word in feature]# 将列表中所有的分词拼接成一个字符串feature_str = ' '.join(wordlist) # 标题分析font_path = r'./simhei.ttf'shoes_box_jpg = imageio.imread('./home.jpg')wc=wordcloud.wordcloud( background_color='black', mask=shoes_box_jpg, font_path = font_path, min_font_size=5, max_font_size=50, width=260, height=260,)wc.generate(feature_str)plt.figure(figsize=(10, 8), dpi=100)plt.imshow(wc)plt.axis('off')plt.savefig('标题提取关键词') |
8.13 筛选价格在4000到5000，联想品牌、处理器是i5、屏幕大小在15寸以上的数据并查看价格
|
1
2
3
4
5
6
7
8
|
df_jd_query = df_jp.loc[(df_jp['computer_price'] <=5000) & (df_jp['computer_price']>=4000) & (df_jp['computer_p_shop']=="联想") & (df_jp['computer_intel']=="i5") & (df_jp['computer_in']=="15.0英寸-15.9英寸"), :].copy()plt.figure(figsize=(20, 10), dpi=100)ax = sns.barplot(x='computer_sku', y='computer_price', data=df_jd_query)ax.set_xlabel('联想品牌sku')ax.set_ylabel('价格')ax.set_title('酷睿i5处理器屏幕15寸以上各sku的价格')boxplot_fig = ax.get_figure()boxplot_fig.savefig('酷睿i5处理器屏幕15寸以上各sku的价格.png', dpi=400) |
8.14 筛选价格在4000到5000，戴尔品牌、处理器是i7、屏幕大小在15寸以上的数据并查看价格
|
1
2
3
4
5
6
7
8
|
df_jp_daier = df_jp.loc[(df_jp['computer_price'] <=5000) & (df_jp['computer_price']>=4000) & (df_jp['computer_p_shop']=="戴尔") & (df_jp['computer_intel']=="i7") & (df_jp['computer_in']=="15.0英寸-15.9英寸"), :].copy()plt.figure(figsize=(10, 8), dpi=100)ax = sns.barplot(x='computer_sku', y='computer_price', data=df_jp_daier)ax.set_xlabel('戴尔品牌sku')ax.set_ylabel('价格')ax.set_title('酷睿i7处理器屏幕15寸以上各sku的价格')boxplot_fig = ax.get_figure()boxplot_fig.savefig('酷睿i7处理器屏幕15寸以上各sku的价格.png', dpi=400) |
8.15 不同intel处理器品牌的价格
|
1
2
3
4
5
6
7
|
plt.figure(figsize=(10, 8), dpi=100)ax = sns.barplot(x='computer_p_shop', y='computer_price', data=df_jp, hue='computer_intel')ax.set_xlabel('品牌')ax.set_ylabel('价格')ax.set_title('不同酷睿处理器品牌的价格')boxplot_fig = ax.get_figure()boxplot_fig.savefig('不同酷睿处理器品牌的价格.png', dpi=400) |
8.16 不同尺寸品牌的价格
|
1
2
3
4
5
6
7
|
plt.figure(figsize=(10, 8), dpi=100)ax = sns.barplot(x='computer_p_shop', y='computer_price', data=df_jp, hue='computer_in')ax.set_xlabel('品牌')ax.set_ylabel('价格')ax.set_title('不同尺寸品牌的价格')boxplot_fig = ax.get_figure()boxplot_fig.savefig('不同尺寸品牌的价格.png', dpi=400) |
以上就是python基于scrapy爬取京东笔记本电脑数据并进行简单处理和分析的详细内容，更多关于python 爬取京东数据的资料请关注服务器之家其它相关文章！
原文链接：https://blog.csdn.net/weixin_45920625/article/details/115673622
