python通過urllib2爬網(wǎng)頁(yè)上種子下載示例

更新時(shí)間：2014年02月24日 14:43:33 作者：

這篇文章主要介紹了通過urllib2、re模塊抓種子下載的示例,需要的朋友可以參考下

通過urllib2、re模塊抓種子

思路

1.用程序登錄論壇(如果需要登錄才能訪問的版塊)

2.訪問指定版塊

3.遍歷帖子（先取指定頁(yè)，再遍歷頁(yè)面所有帖子的url）

4.循環(huán)訪問所有帖子url，從帖子頁(yè)面代碼中取種子下載地址（通過正則表達(dá)式或第三方頁(yè)面解析庫(kù)）

5.訪問種子頁(yè)面下載種子

復(fù)制代碼代碼如下:

import urllib
import urllib2
import cookielib
import re
import sys
import os

# site is website address | fid is part id
site = "http://xxx.yyy.zzz/"
source = "thread0806.php?fid=x&search=&page="

btSave = "./clyzwm/"
if os.path.isdir(btSave):
print btSave + " existing"
else:
os.mkdir(btSave)

logfile = "./clyzwm/down.log"
errorfile = "./clyzwm/error.log"
sucfile = "./clyzwm/sucess.log"

headers = {'User-Agent' : 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36',
'Referer' : 'http://xxx.yyy.zzz/'}

def btDown(url, dirPath):
logger(logfile, "download file : " + url)
try:
  #pageCode = urllib2.urlopen(url).read()
  #print pageCode
  btStep1 = re.findall('http://[\w]+\.[\w]+\.[\w]{0,4}/[\w]{2,6}\.php\?[\w]{2,6}=([\w]+)', url, re.I)
  #print btStep1
  if len(btStep1)>0:
   ref = btStep1[0]
   downsite = ""
   downData = {}
   if len(ref)>20:
    downsite = re.findall('http://www.[\w]+\.[\w]+/', url)[0]
    downsite = downsite + "download.php"
    reff = re.findall('input\stype=\"hidden\"\sname=\"reff\"\svalue=\"([\w=]+)\"', urllib2.urlopen(url).read(), re.I)[0]
    downData = {'ref': ref, 'reff':reff, 'submit':'download'}
   else:
    downsite = "http://www.downhh.com/download.php"
    downData = {'ref': ref, 'rulesubmit':'download'}
   #print "bt site - " + downsite + "\n downData:"
   #print downData
   downData = urllib.urlencode(downData)
   downReq = urllib2.Request(downsite, downData)
   downReq.add_header('User-Agent','Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36')
   downPost = urllib2.urlopen(downReq)
   stream = downPost.read(-1)
   if (len(stream) > 1000):
    downPost.close()
    name = btStep1[0]+ ".torrent"
    fw = open(dirPath + name, 'w')
    fw.write(stream)
    fw.close()
    logger(sucfile, url+"\n")
   else:
    logger(errorfile, url+"\n")
except urllib2.URLError, e:
  print e.reason

def logger(logfile, msg):
print msg
fw = open(logfile, 'a')
fw.write(msg)
fw.close()

for i in range(1, 1000):
logger(logfile, "\n\n\n@ page " + str(i) + " ...")
part = site + source + str(i)

content = urllib2.urlopen(part).read()
content = content.decode('gbk').encode('utf8')
#print content

pages = re.findall('<a\s+href=\"(htm_data/[\d]+/[\d]+/[\d]+\.html).*?<\/a>', content,re.I)
#print pages

for page in pages:
  page = site + page;
  #logger(logfile, "\n# visiting " + page + " ...")
  pageCode = urllib2.urlopen(page).read()
  #print pageCode
  zzJump = re.findall('http://www.viidii.info/\?http://[\w]+/[\w]+\?[\w]{2,6}=[\w]+' ,pageCode)
  #zzJump = re.findall('http://www.viidii.info/\?http://[\w/\?=]*', pageCode)
  if len(zzJump) > 0:
   zzJump = zzJump[0]
   #print "- jump page - " + zzJump
   pageCode = urllib2.urlopen(page).read()
   zzPage = re.findall('http://[\w]+\.[\w]+\.[\w]+/link[\w]?\.php\?[\w]{2,6}=[\w]+' ,pageCode)
   if len(zzPage) > 0:
    zzPage = zzPage[0]
    logger(logfile, "\n- zhongzi page -" + zzPage)
    btDown(zzPage, btSave)
   else:
    logger(logfile, "\n. NOT FOUND .")
  else:
   logger(logfile, "\n... NOT FOUND ...")
  zzPage = re.findall('http://[\w]+\.[\w]+\.[\w]+/link[\w]?\.php\?ref=[\w]+' ,pageCode)