# -*- coding: utf-8 -*- 
""" 
Crawling pictures by selenium and urllib
url: 虎撲 馬刺 http://photo.hupu.com/nba/tag/%E9%A9%AC%E5%88%BA
url: 虎撲 陳露 http://photo.hupu.com/nba/tag/%E9%99%88%E9%9C%B2
Created on 2015-10-24
@author: Eastmount CSDN 
""" 
 
import time   
import re   
import os 
import sys 
import urllib 
import shutil 
import datetime 
from selenium import webdriver  
from selenium.webdriver.common.keys import Keys  
import selenium.webdriver.support.ui as ui  
from selenium.webdriver.common.action_chains import ActionChains 
 
#Open PhantomJS 
driver = webdriver.PhantomJS(executable_path="G:\phantomjs-1.9.1-windows\phantomjs.exe")
#driver = webdriver.Firefox() 
wait = ui.WebDriverWait(driver,10) 
 
#Download one Picture By urllib 
def loadPicture(pic_url, pic_path): 
 pic_name = os.path.basename(pic_url) #刪除路徑獲取圖片名字
 pic_name = pic_name.replace('*','') #去除'*' 防止錯誤 invalid mode ('wb') or filename
 urllib.urlretrieve(pic_url, pic_path + pic_name)
 
 
#爬取具體的圖片及下一張
def getScript(elem_url, path, nums):
 try:
  #由于鏈接 http://photo.hupu.com/nba/p29556-1.html
  #只需拼接 http://..../p29556-數(shù)字.html 省略了自動點擊"下一張"操作
  count = 1
  t = elem_url.find(r'.html')
  while (count <= nums):
   html_url = elem_url[:t] + '-' + str(count) + '.html'
   #print html_url
   '''
   driver_pic.get(html_url)
   elem = driver_pic.find_element_by_xpath("http://div[@class='pic_bg']/div/img")
   url = elem.get_attribute("src")
   '''
   #采用正則表達(dá)式獲取第3個<div></div> 再獲取圖片URL進(jìn)行下載
   content = urllib.urlopen(html_url).read()
   start = content.find(r'<div class="flTab">')
   end = content.find(r'<div class="comMark" style>')
   content = content[start:end]
   div_pat = r'<div.*?>(.*?)<\/div>'
   div_m = re.findall(div_pat, content, re.S|re.M)
   #print div_m[2]
   link_list = re.findall(r"(?<=href=\").+?(?=\")|(?<=href=\').+?(?=\')", div_m[2])
   #print link_list
   url = link_list[0] #僅僅一條url鏈接
   loadPicture(url, path)
   count = count + 1

 except Exception,e: 
  print 'Error:',e 
 finally: 
  print 'Download ' + str(count) + ' pictures\n' 
 
  
#爬取主頁圖片集的URL和主題 
def getTitle(url): 
 try: 
  #爬取URL和標(biāo)題 
  count = 0 
  print 'Function getTitle(key,url)' 
  driver.get(url) 
  wait.until(lambda driver: driver.find_element_by_xpath("http://div[@class='piclist3']"))
  print 'Title: ' + driver.title + '\n'
  
  #縮略圖片url(此處無用) 圖片數(shù)量 標(biāo)題(文件名) 注意順序
  elem_url = driver.find_elements_by_xpath("http://a[@class='ku']/img")
  elem_num = driver.find_elements_by_xpath("http://div[@class='piclist3']/table/tbody/tr/td/dl/dd[1]")
  elem_title = driver.find_elements_by_xpath("http://div[@class='piclist3']/table/tbody/tr/td/dl/dt/a")
  for url in elem_url: 
   pic_url = url.get_attribute("src")
   html_url = elem_title[count].get_attribute("href")
   print elem_title[count].text
   print html_url 
   print pic_url
   print elem_num[count].text
   
   #創(chuàng)建圖片文件夾
   path = "E:\\Picture_HP\\" + elem_title[count].text + "\\"
   m = re.findall(r'(\w*[0-9]+)\w*', elem_num[count].text) #爬蟲圖片張數(shù)
   nums = int(m[0])
   count = count + 1 
   if os.path.isfile(path):   #Delete file 
    os.remove(path) 
   elif os.path.isdir(path):  #Delete dir 
    shutil.rmtree(path, True) 
   os.makedirs(path)    #create the file directory 
   getScript(html_url, path, nums) #visit pages
     
 except Exception,e: 
  print 'Error:',e 
 finally: 
  print 'Find ' + str(count) + ' pages with key\n' 
  
#Enter Function 
def main(): 
 #Create Folder 
 basePathDirectory = "E:\\Picture_HP" 
 if not os.path.exists(basePathDirectory): 
  os.makedirs(basePathDirectory) 
 
 #Input the Key for search str=>unicode=>utf-8 
 key = raw_input("Please input a key: ").decode(sys.stdin.encoding) 
 print 'The key is : ' + key 
 
 #Set URL List Sum:1-2 Pages 
 print 'Ready to start the Download!!!\n\n' 
 starttime = datetime.datetime.now() 
 num=1 
 while num<=1:
  #url = 'http://photo.hupu.com/nba/tag/%E9%99%88%E9%9C%B2?p=2&o=1'
  url = 'http://photo.hupu.com/nba/tag/%E9%A9%AC%E5%88%BA'  
  print '第'+str(num)+'頁','url:'+url 
  #Determine whether the title contains key 
  getTitle(url) 
  time.sleep(2) 
  num = num + 1 
 else: 
  print 'Download Over!!!' 
 
 #get the runtime 
 endtime = datetime.datetime.now() 
 print 'The Running time : ',(endtime - starttime).seconds 
   
main()