scrapydownloadertest  # 項(xiàng)目文件夾
    │  items.py       # 定義爬取結(jié)果存儲(chǔ)的數(shù)據(jù)結(jié)構(gòu)
    │  middlewares.py  # 中間件（可以理解java的過濾器攔截器）
    │  pipelines.py   # 數(shù)據(jù)管道，對(duì)獲取到的數(shù)據(jù)做操作
    │  settings.py   # 項(xiàng)目的配置文件
    │  __init__.py   # 初始化邏輯
    │
    ├─spiders  # 放置 Spiders 的文件夾
    │  │  httpProxyIp.py   # 爬取到結(jié)果后的處理類
    │  │  __init__.py    # spider初始化邏輯
scrapy.py  

# Scrapy 內(nèi)置的 Downloader Middleware 為 Scrapy 供了基礎(chǔ)的功能，
# 定義一個(gè)類，其中（object）可以不寫，效果一樣
class SimpleProxyMiddleware(object):
    # 聲明一個(gè)數(shù)組
    proxyList = ['http://218.75.158.153:3128','http://188.226.141.61:8080']
    
    # Downloader Middleware的核心方法，只有實(shí)現(xiàn)了其中一個(gè)或多個(gè)方法才算自定義了一個(gè)Downloader Middleware
    def process_request(self, request, spider):
        # 隨機(jī)從其中選擇一個(gè)，并去除左右兩邊空格
        proxy = random.choice(self.proxyList).strip()
        # 打印結(jié)果出來觀察
        print("this is request ip:" + proxy)
        # 設(shè)置request的proxy屬性的內(nèi)容為代理ip
        request.meta['proxy'] = proxy

    # Downloader Middleware的核心方法，只有實(shí)現(xiàn)了其中一個(gè)或多個(gè)方法才算自定義了一個(gè)Downloader Middleware
    def process_response(self, request, response, spider):
        # 請(qǐng)求失敗不等于200
        if response.status != 200:
            # 重新選擇一個(gè)代理ip
            proxy = random.choice(self.proxyList).strip()
            print("this is response ip:" + proxy)
            # 設(shè)置新的代理ip內(nèi)容
            request.mete['proxy'] = proxy
            return request
        return response

# -*- coding: utf-8 -*-
import scrapy

class HttpproxyipSpider(scrapy.Spider):
    name = 'httpProxyIp'
    allowed_domains = ['icanhazip.com']
    start_urls = ['http://icanhazip.com/']

    def parse(self, response):
        pass

# -*- coding: utf-8 -*-
import scrapy
from scrapy.cmdline import execute

class HttpproxyipSpider(scrapy.Spider):
    # spider 任務(wù)名
    name = 'httpProxyIp'
    # 允許訪問的域名
    allowed_domains = ['icanhazip.com']
    # 起始爬取的url
    start_urls = ['http://icanhazip.com/']

    # spider 爬蟲解析的方法，關(guān)于內(nèi)容的解析都在這里完成; self表示實(shí)例的引用， response爬蟲的結(jié)果
    def parse(self, response):
        print('代理后的ip: ', response.text)

# 這個(gè)是main函數(shù)也是整個(gè)程序入口的慣用寫法
if __name__ == '__main__':
    execute(['scrapy', 'crawl', 'httpbin'])

DOWNLOADER_MIDDLEWARES = {
    # 注釋掉之前的例子改用AbuyunProxyMiddleware
    # 'scrapydownloadertest.middlewares.SimpleProxyMiddleware': 100,
    'scrapydownloadertest.middlewares.AbuyunProxyMiddleware': 100,
}