from scrapy.downloadermiddlewares.retry import RetryMiddleware

class RetryMiddleware(object):

  # IOError is raised by the HttpCompression middleware when trying to
  # decompress an empty response
  EXCEPTIONS_TO_RETRY = (defer.TimeoutError, TimeoutError, DNSLookupError,
              ConnectionRefusedError, ConnectionDone, ConnectError,
              ConnectionLost, TCPTimedOutError, ResponseFailed,
              IOError, TunnelError)

  def __init__(self, settings):
    if not settings.getbool('RETRY_ENABLED'):
      raise NotConfigured
    self.max_retry_times = settings.getint('RETRY_TIMES')
    self.retry_http_codes = set(int(x) for x in settings.getlist('RETRY_HTTP_CODES'))
    self.priority_adjust = settings.getint('RETRY_PRIORITY_ADJUST')

  @classmethod
  def from_crawler(cls, crawler):
    return cls(crawler.settings)

  def process_response(self, request, response, spider):
    if request.meta.get('dont_retry', False):
      return response
    if response.status in self.retry_http_codes:
      reason = response_status_message(response.status)
      return self._retry(request, reason, spider) or response
    return response

  def process_exception(self, request, exception, spider):
    if isinstance(exception, self.EXCEPTIONS_TO_RETRY) \
        and not request.meta.get('dont_retry', False):
      return self._retry(request, exception, spider)

  def _retry(self, request, reason, spider):
    retries = request.meta.get('retry_times', 0) + 1

    retry_times = self.max_retry_times

    if 'max_retry_times' in request.meta:
      retry_times = request.meta['max_retry_times']

    stats = spider.crawler.stats
    if retries <= retry_times:
      logger.debug("Retrying %(request)s (failed %(retries)d times): %(reason)s",
             {'request': request, 'retries': retries, 'reason': reason},
             extra={'spider': spider})
      retryreq = request.copy()
      retryreq.meta['retry_times'] = retries
      retryreq.dont_filter = True
      retryreq.priority = request.priority + self.priority_adjust

      if isinstance(reason, Exception):
        reason = global_object_name(reason.__class__)

      stats.inc_value('retry/count')
      stats.inc_value('retry/reason_count/%s' % reason)
      return retryreq
    else:
      stats.inc_value('retry/max_reached')
      logger.debug("Gave up retrying %(request)s (failed %(retries)d times): %(reason)s",
             {'request': request, 'retries': retries, 'reason': reason},
             extra={'spider': spider})

RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408]

from twisted.internet import defer
from twisted.internet.error import TimeoutError, DNSLookupError, \
  ConnectionRefusedError, ConnectionDone, ConnectError, \
  ConnectionLost, TCPTimedOutError
from scrapy.http import HtmlResponse
from twisted.web.client import ResponseFailed
from scrapy.core.downloader.handlers.http11 import TunnelError

class ProcessAllExceptionMiddleware(object):
  ALL_EXCEPTIONS = (defer.TimeoutError, TimeoutError, DNSLookupError,
           ConnectionRefusedError, ConnectionDone, ConnectError,
           ConnectionLost, TCPTimedOutError, ResponseFailed,
           IOError, TunnelError)
  def process_response(self,request,response,spider):
    #捕獲狀態(tài)碼為40x/50x的response
    if str(response.status).startswith('4') or str(response.status).startswith('5'):
      #隨意封裝，直接返回response，spider代碼中根據(jù)url==''來處理response
      response = HtmlResponse(url='')
      return response
    #其他狀態(tài)碼不處理
    return response
  def process_exception(self,request,exception,spider):
    #捕獲幾乎所有的異常
    if isinstance(exception, self.ALL_EXCEPTIONS):
      #在日志中打印異常類型
      print('Got exception: %s' % (exception))
      #隨意封裝一個(gè)response，返回給spider
      response = HtmlResponse(url='exception')
      return response
    #打印出未捕獲到的異常
    print('not contained exception: %s'%exception)

class TESTSpider(scrapy.Spider):
  name = 'TEST'
  allowed_domains = ['TTTTT.com']
  start_urls = ['http://www.TTTTT.com/hypernym/?q=']
  custom_settings = {
    'DOWNLOADER_MIDDLEWARES': {
      'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None,
      'TESTSpider.middlewares.ProcessAllExceptionMiddleware': 120,
    },
    'DOWNLOAD_DELAY': 1, # 延時(shí)最低為2s
    'AUTOTHROTTLE_ENABLED': True, # 啟動(dòng)[自動(dòng)限速]
    'AUTOTHROTTLE_DEBUG': True, # 開啟[自動(dòng)限速]的debug
    'AUTOTHROTTLE_MAX_DELAY': 10, # 設(shè)置最大下載延時(shí)
    'DOWNLOAD_TIMEOUT': 15,
    'CONCURRENT_REQUESTS_PER_DOMAIN': 4 # 限制對(duì)該網(wǎng)站的并發(fā)請(qǐng)求數(shù)
  }
  def parse(self, response):
    if not response.url: #接收到url==''時(shí)
      print('500')
      yield TESTItem(key=response.meta['key'], _str=500, alias='')
    elif 'exception' in response.url:
      print('exception')
      yield TESTItem(key=response.meta['key'], _str='EXCEPTION', alias='')