python基於scrapy爬取京東筆記本電腦數據並進行簡單處理和分析

pip install scrapy fake-useragent requests selenium virtualenv -i https://pypi.douban.com/simple

virtualenv .venv

scrapy startproject 項目名稱

scrapy genspider 爬蟲名稱 url

import re
import scrapy


from lianjia.items import jd_detailItem


class JiComputerDetailSpider(scrapy.Spider):
    name = 'ji_computer_detail'
    allowed_domains = ['search.jd.com', 'item.jd.com']
    start_urls = [
        'https://search.jd.com/Search?keyword=%E7%AC%94%E8%AE%B0%E6%9C%AC%E7%94%B5%E8%84%91&suggest=1.def.0.base&wq=%E7%AC%94%E8%AE%B0%E6%9C%AC%E7%94%B5%E8%84%91&page=1&s=1&click=0']

    def parse(self, response):
        lls = response.xpath('//ul[@class="gl-warp clearfix"]/li')
        for ll in lls:
            item = jd_detailItem()
            computer_price = ll.xpath('.//div[@class="p-price"]/strong/i/text()').extract_first()
            computer_commit = ll.xpath('.//div[@class="p-commit"]/strong/a/text()').extract_first()
            computer_p_shop = ll.xpath('.//div[@class="p-shop"]/span/a/text()').extract_first()
            item['computer_price'] = computer_price
            item['computer_commit'] = computer_commit
            item['computer_p_shop'] = computer_p_shop
            meta = {
                'item': item
            }
            shop_detail_url = ll.xpath('.//div[@class="p-img"]/a/@href').extract_first()
            shop_detail_url = 'https:' + shop_detail_url
            yield scrapy.Request(url=shop_detail_url, callback=self.detail_parse, meta=meta)
        for i in range(2, 200, 2):
            next_page_url = f'https://search.jd.com/Search?keyword=%E7%AC%94%E8%AE%B0%E6%9C%AC%E7%94%B5%E8%84%91&suggest=1.def.0.base&wq=%E7%AC%94%E8%AE%B0%E6%9C%AC%E7%94%B5%E8%84%91&page={i}&s=116&click=0'
            yield scrapy.Request(url=next_page_url, callback=self.parse)

    def detail_parse(self, response):
        item = response.meta.get('item')
        computer_sku = response.xpath('//a[@class="notice J-notify-sale"]/@data-sku').extract_first()
        item['computer_sku'] = computer_sku
        computer_title = response.xpath('//div[@class="sku-name"]/text()').extract_first().strip()
        computer_title = ''.join(re.findall('\S', computer_title))
        item['computer_title'] = computer_title
        computer_detail = response.xpath('string(//ul[@class="parameter2 p-parameter-list"])').extract_first().strip()
        computer_detail = ''.join(re.findall('\S', computer_detail))
        item['computer_detail'] = computer_detail
        yield item

class jd_detailItem(scrapy.Item):
    # define the fields for your item here like:
    computer_sku = scrapy.Field()
    computer_price = scrapy.Field()
    computer_title = scrapy.Field()
    computer_commit = scrapy.Field()
    computer_p_shop = scrapy.Field()
    computer_detail = scrapy.Field()

import random


from fake_useragent import UserAgent
ua = UserAgent()
USER_AGENT = ua.random
ROBOTSTXT_OBEY = False
DOWNLOAD_DELAY = random.uniform(0.5, 1)
DOWNLOADER_MIDDLEWARES = {
    'lianjia.middlewares.jdDownloaderMiddleware': 543
}
ITEM_PIPELINES = {
    'lianjia.pipelines.jd_csv_Pipeline': 300
}

class jd_csv_Pipeline:
    # def process_item(self, item, spider):
    #     return item
    def open_spider(self, spider):
        self.fp = open('./jd_computer_message.xlsx', mode='w+', encoding='utf-8')
        self.fp.write('computer_sku\tcomputer_title\tcomputer_p_shop\tcomputer_price\tcomputer_commit\tcomputer_detail\n')

    def process_item(self, item, spider):
        # 寫入文件
        try:
            line = '\t'.join(list(item.values())) + '\n'
            self.fp.write(line)
            return item
        except:
            pass

    def close_spider(self, spider):
        # 關閉文件
        self.fp.close()

class jdDownloaderMiddleware:
    def process_request(self, request, spider):
        # 判斷是否是ji_computer_detail的爬蟲
        # 判斷是否是首頁
        if spider.name == 'ji_computer_detail' and re.findall(f'.*(item.jd.com).*', request.url) == []:
            options = ChromeOptions()
            options.add_argument("--headless")
            driver = webdriver.Chrome(options=options)
            driver.get(request.url)
            for i in range(0, 15000, 5000):
                driver.execute_script(f'window.scrollTo(0, {i})')
                time.sleep(0.5)
            body = driver.page_source.encode()
            time.sleep(1)
            return HtmlResponse(url=request.url, body=body, request=request)
        return None

!pip install seaborn jieba wordcloud PIL  -i https://pypi.douban.com/simple

import re
import os
import jieba
import wordcloud
import pandas as pd
import numpy as np
from PIL import Image
import seaborn as sns
from docx import Document
from docx.shared import Inches
import matplotlib.pyplot as plt
from pandas import DataFrame,Series

sns.set_style('darkgrid')
plt.rcParams['font.sans-serif'] = ['SimHei']
plt.rcParams['axes.unicode_minus'] = False

df_jp = pd.read_excel('./jd_shop.xlsx')

def convert_one(s):
    if re.findall(f'.*?(i5).*', str(s)) != []:
        return re.findall(f'.*?(i5).*', str(s))[0]
    elif re.findall(f'.*?(i7).*', str(s)) != []:
        return re.findall(f'.*?(i7).*', str(s))[0]
    elif re.findall(f'.*?(i9).*', str(s)) != []:
        return re.findall(f'.*?(i9).*', str(s))[0]
df_jp['computer_intel'] = df_jp['computer_detail'].map(convert_one)

def convert_two(s):
    if re.findall(f'.*?(\d+\.\d+英寸-\d+\.\d+英寸).*', str(s)) != []:
        return re.findall(f'.*?(\d+\.\d+英寸-\d+\.\d+英寸).*', str(s))[0]
df_jp['computer_in'] = df_jp['computer_detail'].map(convert_two)

def convert_three(s):
    if re.findall(f'(\d+)萬+', str(s)) != []:
        number = int(re.findall(f'(\d+)萬+', str(s))[0]) * 10000
        return number
    elif re.findall(f'(\d+)+', str(s)) != []:
        number = re.findall(f'(\d+)+', str(s))[0]
        return number
df_jp['computer_commit'] = df_jp['computer_commit'].map(convert_three)

def find_computer(name, s):
    sr = re.findall(f'.*({name}).*', str(s))[0]
    return sr
def convert(s):
    if re.findall(f'.*(聯想).*', str(s)) != []:
        return find_computer('聯想', s)
    elif re.findall(f'.*(惠普).*', str(s)) != []:
        return find_computer('惠普', s)
    elif re.findall(f'.*(華為).*', str(s)) != []:
        return find_computer('華為', s)
    elif re.findall(f'.*(戴爾).*', str(s)) != []:
        return find_computer('戴爾', s)
    elif re.findall(f'.*(華碩).*', str(s)) != []:
        return find_computer('華碩', s)
    elif re.findall(f'.*(小米).*', str(s)) != []:
        return find_computer('小米', s)
    elif re.findall(f'.*(榮耀).*', str(s)) != []:
        return find_computer('榮耀', s)
    elif re.findall(f'.*(神舟).*', str(s)) != []:
        return find_computer('神舟', s)
    elif re.findall(f'.*(外星人).*', str(s)) != []:
        return find_computer('外星人', s)
df_jp['computer_p_shop'] = df_jp['computer_p_shop'].map(convert)

for n in ['computer_price', 'computer_commit', 'computer_p_shop', 'computer_sku', 'computer_detail', 'computer_intel', 'computer_in']:
    index_ls = df_jp[df_jp[[n]].isnull().any(axis=1)==True].index
    df_jp.drop(index=index_ls, inplace=True)

plt.figure(figsize=(10, 8), dpi=100)
ax = sns.barplot(x='computer_p_shop', y='computer_price', data=df_jp.groupby(by='computer_p_shop')[['computer_price']].mean().reset_index())
for index,row in df_jp.groupby(by='computer_p_shop')[['computer_price']].mean().reset_index().iterrows():
    ax.text(row.name,row['computer_price'] + 2,round(row['computer_price'],2),color="black",ha="center")
ax.set_xlabel('品牌')
ax.set_ylabel('平均價格')
ax.set_title('各品牌平均價格')
boxplot_fig = ax.get_figure()
boxplot_fig.savefig('各品牌平均價格.png', dpi=400)

plt.figure(figsize=(10, 8), dpi=100)
ax = sns.boxenplot(x='computer_p_shop', y='computer_price', data=df_jp.query('computer_price>500'))
ax.set_xlabel('品牌')
ax.set_ylabel('價格區間')
ax.set_title('各品牌價格區間')
boxplot_fig = ax.get_figure()
boxplot_fig.savefig('各品牌價格區間.png', dpi=400)

df_jp['computer_commit'] = df_jp['computer_commit'].astype('int64')
ax = sns.jointplot(x="computer_commit", y="computer_price", data=df_jp, kind="reg", truncate=False,color="m", height=10)
ax.fig.savefig('評論數與價格的關系.png')

import imageio

# 將特征轉換為列表
ls = df_jp['computer_title'].to_list()
# 替換非中英文的字符
feature_points = [re.sub(r'[^a-zA-Z\u4E00-\u9FA5]+',' ',str(feature)) for feature in ls]
# 讀取停用詞
stop_world = list(pd.read_csv('./百度停用詞表.txt', engine='python', encoding='utf-8', names=['stopwords'])['stopwords'])
feature_points2 = []
for feature in feature_points:  # 遍歷每一條評論
    words = jieba.lcut(feature) # 精確模式，沒有冗餘.對每一條評論進行jieba分詞
    ind1 = np.array([len(word) > 1 for word in words])  # 判斷每個分詞的長度是否大於1
    ser1 = pd.Series(words)
    ser2 = ser1[ind1] # 篩選分詞長度大於1的分詞留下
    ind2 = ~ser2.isin(stop_world)  # 註意取反負號
    ser3 = ser2[ind2].unique()  # 篩選出不在停用詞表的分詞留下,並去重
    if len(ser3) > 0:
        feature_points2.append(list(ser3))
# 將所有分詞存儲到一個列表中
wordlist = [word for feature in feature_points2 for word in feature]
# 將列表中所有的分詞拼接成一個字符串
feature_str =  ' '.join(wordlist)   
# 標題分析
font_path = r'./simhei.ttf'
shoes_box_jpg = imageio.imread('./home.jpg')
wc=wordcloud.WordCloud(
    background_color='black',
    mask=shoes_box_jpg,
    font_path = font_path,
    min_font_size=5,
    max_font_size=50,
    width=260,
    height=260,
)
wc.generate(feature_str)
plt.figure(figsize=(10, 8), dpi=100)
plt.imshow(wc)
plt.axis('off')
plt.savefig('標題提取關鍵詞')

df_jd_query = df_jp.loc[(df_jp['computer_price'] <=5000) & (df_jp['computer_price']>=4000) & (df_jp['computer_p_shop']=="聯想") & (df_jp['computer_intel']=="i5") & (df_jp['computer_in']=="15.0英寸-15.9英寸"), :].copy()
plt.figure(figsize=(20, 10), dpi=100)
ax = sns.barplot(x='computer_sku', y='computer_price', data=df_jd_query)
ax.set_xlabel('聯想品牌SKU')
ax.set_ylabel('價格')
ax.set_title('酷睿i5處理器屏幕15寸以上各SKU的價格')
boxplot_fig = ax.get_figure()
boxplot_fig.savefig('酷睿i5處理器屏幕15寸以上各SKU的價格.png', dpi=400)

df_jp_daier = df_jp.loc[(df_jp['computer_price'] <=5000) & (df_jp['computer_price']>=4000) & (df_jp['computer_p_shop']=="戴爾") & (df_jp['computer_intel']=="i7") & (df_jp['computer_in']=="15.0英寸-15.9英寸"), :].copy()
plt.figure(figsize=(10, 8), dpi=100)
ax = sns.barplot(x='computer_sku', y='computer_price', data=df_jp_daier)
ax.set_xlabel('戴爾品牌SKU')
ax.set_ylabel('價格')
ax.set_title('酷睿i7處理器屏幕15寸以上各SKU的價格')
boxplot_fig = ax.get_figure()
boxplot_fig.savefig('酷睿i7處理器屏幕15寸以上各SKU的價格.png', dpi=400)

plt.figure(figsize=(10, 8), dpi=100)
ax = sns.barplot(x='computer_p_shop', y='computer_price', data=df_jp, hue='computer_intel')
ax.set_xlabel('品牌')
ax.set_ylabel('價格')
ax.set_title('不同酷睿處理器品牌的價格')
boxplot_fig = ax.get_figure()
boxplot_fig.savefig('不同酷睿處理器品牌的價格.png', dpi=400)

plt.figure(figsize=(10, 8), dpi=100)
ax = sns.barplot(x='computer_p_shop', y='computer_price', data=df_jp, hue='computer_in')
ax.set_xlabel('品牌')
ax.set_ylabel('價格')
ax.set_title('不同尺寸品牌的價格')
boxplot_fig = ax.get_figure()
boxplot_fig.savefig('不同尺寸品牌的價格.png', dpi=400)