url = 'http://i.waimai.meituan.com/openh5/poi/filterconditions?_=1557367197922'

form_data = {
 'initialLat': '25.618626',
 'initialLng': '105.644569',
 'actualLat': '25.618626',
 'actualLng': '105.644569',
 'geoType': '2',
 'wm_latitude': '25618626',
 'wm_longitude': '105644569',
 'wm_actual_latitude': '25618626',
 'wm_actual_longitude': '105644569'
}

meta = {
 'lat': form_data.get('initialLat'),
 'lng': form_data.get('initialLng'),
 'lat2': form_data.get('wm_latitude'),
 'lng2': form_data.get('wm_longitude'),
 'province': '**省',
 'city': '**市',
 'area': '**區(qū)'
}

class RedisSpider(RedisMixin, Spider):
 @classmethod
 def from_crawler(self, crawler, *args, **kwargs):
 obj = super(RedisSpider, self).from_crawler(crawler, *args, **kwargs)
 obj.setup_redis(crawler)
 return obj 

def start_requests(self):
 return self.next_requests()

def next_requests(self):
 """Returns a request to be scheduled or none."""
 use_set = self.settings.getbool('REDIS_START_URLS_AS_SET',    defaults.START_URLS_AS_SET)
 fetch_one = self.server.spop if use_set else self.server.lpop
 # XXX: Do we need to use a timeout here?
 found = 0
 # TODO: Use redis pipeline execution.
 while found < self.redis_batch_size: # 每次讀取的量
  data = fetch_one(self.redis_key) # 從redis中讀取一條記錄
  if not data:
   # Queue empty.
   break
  req = self.make_request_from_data(data) # 根據(jù)從redis中讀取的記錄，實例化一個request
  if req:
   yield req
  found += 1
  else:
   self.logger.debug("Request not made from data: %r", data)
 
 if found:
  self.logger.debug("Read %s requests from '%s'", found, self.redis_key)

def make_request_from_data(self, data):
 url = bytes_to_str(data, self.redis_encoding)
 return self.make_requests_from_url(url) # 這是重點，圈起來，要考

from scrapy import FormRequest
from scrapy_redis.spiders import RedisSpider
 
 
class MeituanSpider(RedisSpider):
 """
 此處省略若干行
 """
 
 def make_request_from_data(self, data):
  """
  重寫make_request_from_data方法，data是scrapy-redis讀取redis中的[url,form_data,meta]，然后發(fā)送post請求
  :param data: redis中都去的請求數(shù)據(jù)，是一個list
  :return: 一個FormRequest對象
 　　"""
  data = json.loads(data)
  url = data.get('url')
  form_data = data.get('form_data')
  meta = data.get('meta')
  return FormRequest(url=url, formdata=form_data, meta=meta, callback=self.parse)

 def parse(self, response):
  pass

def push_start_url_data(request_data):
 """
 將一個完整的request_data推送到redis的start_url列表中
 :param request_data: {'url':url, 'form_data':form_data, 'meta':meta}
 :return:
 """
 r.lpush('meituan:start_urls', request_data)
 
 
if __name__ == '__main__':
 url = 'http://i.waimai.meituan.com/openh5/poi/filterconditions?_=1557367197922'
 form_data = {
  'initialLat': '25.618626',
  'initialLng': '105.644569',
  'actualLat': '25.618626',
  'actualLng': '105.644569',
  'geoType': '2',
  'wm_latitude': '25618626',
  'wm_longitude': '105644569',
  'wm_actual_latitude': '25618626',
  'wm_actual_longitude': '105644569'
 }
 meta = {
  'lat': form_data.get('initialLat'),
  'lng': form_data.get('initialLng'),
  'lat2': form_data.get('wm_latitude'),
  'lng2': form_data.get('wm_longitude'),
  'province': '**省',
  'city': '*市',
  'area': '**區(qū)'
 }
 request_data = {
  'url': url,
  'form_data': form_data,
  'meta': meta
 }
 push_start_url_data(json.dumps(request_data))