怎么用Scrapy构建一个网络爬虫

$ xcode-select --install

# -*- coding: utf-8 -*-

import scrapy

class MyCrawlerItem(scrapy.Item):

    title = scrapy.Field()    # 文章标题

    url = scrapy.Field()      # 文章地址

    summary = scrapy.Field()  # 文章摘要

    pass

# -*- coding: utf-8 -*-

import pymongo

from scrapy.conf import settings

from scrapy.exceptions import DropItem

class MyCrawlerPipeline(object):

    def __init__(self):

        # 设置MongoDB连接

        connection = pymongo.Connection(

            settings['MONGO_SERVER'],

            settings['MONGO_PORT']

        )

        db = connection[settings['MONGO_DB']]

        self.collection = db[settings['MONGO_COLLECTION']]

    # 处理每个被抓取的MyCrawlerItem项

    def process_item(self, item, spider):

        valid = True

        for data in item:

            if not data:  # 过滤掉存在空字段的项

                valid = False

                raise DropItem("Missing {0}!".fORMat(data))

        if valid:

            # 也可以用self.collection.insert(dict(item))，使用upsert可以防止重复项

            self.collection.update({'url': item['url']}, dict(item), upsert=True)

        return item

ITEM_PIPELINES = {

    'my_crawler.pipelines.MyCrawlerPipeline': 300,    # 设置Pipeline，可以多个，值为执行优先级

}

# MongoDB连接信息

MONGO_SERVER = 'localhost'

MONGO_PORT = 27017

MONGO_DB = 'bjhee'

MONGO_COLLECTION = 'articles'

DOWNLOAD_DELAY=2    # 如果网络慢，可以适当加些延迟，单位是秒