python實現scrapy爬蟲每天定時抓取數據的示例代碼

from scrapy import cmdline
cmdline.execute('scrapy crawl mySpider'.split())

# 文件pipelines.py
# 爬蟲啟動時
checkFile = "isRunning.txt"
class myPipeline:
  def open_spider(self, spider):
    self.client = MongoClient('localhost:27017') # 連接Mongodb
    self.db = self.client['mydata']        # 待存儲數據的數據庫mydata
    f = open(checkFile, "w")     # 創建一個文件，代表爬蟲在運行中
    f.close()

# 文件pipelines.py
# 爬蟲正常結束時
checkFile = "isRunning.txt"
class myPipeline:
  def close_spider(self, spider):
    self.client.close()
    isFileExsit = os.path.isfile(checkFile)
    if isFileExsit:
      os.remove(checkFile)

# 在scrapy項目中添加start.py文件，用於啟動爬蟲
from scrapy import cmdline
# 在爬蟲運行過程中，會自動將狀態信息存儲在crawls/storeMyRequest目錄下，支持續爬
cmdline.execute('scrapy crawl mySpider -s JOBDIR=crawls/storeMyRequest'.split())
# Note:若想支持續爬，在ctrl+c終止爬蟲時，隻能按一次，爬蟲在終止時需要進行善後工作，切勿連續多次按ctrl+c

# 文件mySpider.py
class mySpider(CrawlSpider):
  name = "mySpider"
  allowed_domains = ['http://photo.poco.cn/']
  custom_settings = {
    'LOG_LEVEL':'INFO', # 減少Log輸出量，僅保留必要的信息
    # ...... 在爬蟲內部用custom_setting可以讓這個配置信息僅對這一個爬蟲生效
  }

# 文件settings.py
import datetime
BOT_NAME = 'mySpider'
ROBOTSTXT_OBEY = False
startDate = datetime.datetime.now().strftime('%Y%m%d')
LOG_FILE=f"mySpiderlog{startDate}.txt"

# 文件pipelines.py
import datetime
GALANCE=f'galance{datetime.datetime.now().strftime("%Y%m%d")}' # 表名

class myPipeline:
  def open_spider(self, spider):
    self.client = MongoClient('localhost:27017') # 連接Mongodb
    self.db = self.client['mydata']        # 待存儲數據的數據庫mydata

self.db[GALANCE].insert(dict(item))

# 文件run.bat
cd /d F:/newClawer20170831/mySpider
call python main.py
pause

# 文件timerStartDaily.py
from scrapy import cmdline
import datetime
import time
import shutil
import os

recoderDir = r"crawls"  # 這是為瞭爬蟲能夠續爬而創建的目錄，存儲續爬需要的數據
checkFile = "isRunning.txt" # 爬蟲是否在運行的標志

startTime = datetime.datetime.now()
print(f"startTime = {startTime}")

i = 0
miniter = 0
while True:
  isRunning = os.path.isfile(checkFile)
  if not isRunning:            # 爬蟲不在執行，開始啟動爬蟲
    # 在爬蟲啟動之前處理一些事情，清掉JOBDIR = crawls
    isExsit = os.path.isdir(recoderDir) # 檢查JOBDIR目錄crawls是否存在
    print(f"mySpider not running, ready to start. isExsit:{isExsit}")
    if isExsit:
      removeRes = shutil.rmtree(recoderDir) # 刪除續爬目錄crawls及目錄下所有文件
      print(f"At time:{datetime.datetime.now()}, delete res:{removeRes}")
    else:
      print(f"At time:{datetime.datetime.now()}, Dir:{recoderDir} is not exsit.")
    time.sleep(20)
    clawerTime = datetime.datetime.now()
    waitTime = clawerTime - startTime
    print(f"At time:{clawerTime}, start clawer: mySpider !!!, waitTime:{waitTime}")
    cmdline.execute('scrapy crawl mySpider -s JOBDIR=crawls/storeMyRequest'.split())
    break #爬蟲結束之後，退出腳本
  else:
    print(f"At time:{datetime.datetime.now()}, mySpider is running, sleep to wait.")
  i += 1
  time.sleep(600)    # 每10分鐘檢查一次
  miniter += 10
  if miniter >= 1440:  # 等待滿24小時，自動退出監控腳本
    break

# 文件runTimerRunDaily.bat
cd /d F:/newClawer20170831/mySpider
call python timerStartDaily.py
pause