# python 2.7
import requests
from lxml import html
import os

# 獲取首頁(yè)中子頁(yè)的url鏈接
def get_page_urls(url):
  response = requests.get(url).content
  # 通過(guò)lxml的html來(lái)構(gòu)建選擇器
  selector = html.fromstring(response)
  urls = []
  for i in selector.xpath("/html/body//div[@class='feed-item _j_feed_item']/a/@href"):
    urls.append(i)
  return urls

# get title from a child's html(div[@class='title'])
def get_page_a_title(url):
  '''url is ziyouxing's a@href'''
  response = requests.get(url).content
  selector = html.fromstring(response)
  # get xpath by chrome's tool --> /html/body//div[@class='title']/text()
  a_title = selector.xpath("/html/body//div[@class='title']/text()")
  return a_title

# 獲取頁(yè)面選擇器(通過(guò)lxml的html構(gòu)建)
def get_selector(url):
  response = requests.get(url).content
  selector = html.fromstring(response)
  return selector

# 通過(guò)chrome的開(kāi)發(fā)者工具分析html頁(yè)面結(jié)構(gòu)后發(fā)現(xiàn)，我們需要獲取的文本內(nèi)容主要顯示在div[@class='l-topic']和div[@class='p-section']中

# 獲取所需的文本內(nèi)容
 def get_page_content(selector):
   # /html/body/div[2]/div[2]/div[1]/div[@class='l-topic']/p/text()
   page_title = selector.xpath("http://div[@class='l-topic']/p/text()")
   # /html/body/div[2]/div[2]/div[1]/div[2]/div[15]/div[@class='p-section']/text()
   page_content = selector.xpath("http://div[@class='p-section']/text()")
   return page_title,page_content

# 獲取頁(yè)面中的圖片url地址
def get_image_urls(selector):
  imagesrcs = selector.xpath("http://img[@class='_j_lazyload']/@src")
  return imagesrcs

# 獲取圖片的標(biāo)題

def get_image_title(selector, num)
  # num 是從2開(kāi)始的
  url = "/html/body/div[2]/div[2]/div[1]/div[2]/div["+num+"]/span[@class='img-an']/text()"
  if selector.xpath(url) is not None:
    image_title = selector.xpath(url)
  else:
    image_title = "map"+str(num) # 沒(méi)有就起一個(gè)
  return image_title

# 下載圖片

def downloadimages(selector,number):
  '''number是用來(lái)計(jì)數(shù)的'''
  urls = get_image_urls()
  num = 2
  amount = len(urls)
  for url in urls:
    image_title = get_image_title(selector, num)
    filename = "/home/WorkSpace/tour/words/result"+number+"/+"image_title+".jpg"
    if not os.path.exists(filename):
      os.makedirs(filename)
    print('downloading %s image %s' %(number, image_title))
    with open(filename, 'wb') as f:
      f.write(requests.get(url).content)
    num += 1
  print "已經(jīng)下載了%s張圖" %num

# 入口，啟動(dòng)并把獲取的數(shù)據(jù)存入文件中
if __name__ =='__main__':
  url = 'https://www.mafengwo.cn/gonglve/ziyouxing/2033.html'
  urls = get_page_urls(url)
  # turn to get response from html
  number = 1
  for i in urls:
    selector = get_selector(i)
    # download images
    downloadimages(selector,number)
    # get text and write into a file
    page_title, page_content = get_page_content(selector)
    result = page_title+'\n'+page_content+'\n\n'
    path = "/home/WorkSpace/tour/words/result"+num+"/"
    if not os.path.exists(filename):
      os.makedirs(filename)
    filename = path + "num"+".txt"
    with open(filename,'wb') as f:
      f.write(result)
    print result