"""例"""
from urllib import request
response = request.urlopen(r'http://bbs.pinggu.org/') 
#返回狀態(tài) 200證明訪問(wèn)成功
print("返回狀態(tài)碼：  "+str(response.status))
#讀取頁(yè)面信息轉(zhuǎn)換文本并進(jìn)行解碼，如果本身是UTF-8就不要，具體看頁(yè)面格式
#搜索“charset”查看編碼格式
response.read().decode('gbk')[:100]

url = r'http://bbs.pinggu.org/'
headers = {'User-Agent': r'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) '
           r'Chrome/45.0.2454.85 Safari/537.36 115Browser/6.0.3',
           'Referer': r'http://www.lagou.com/zhaopin/Python/?labelWords=label',
           'Connection': 'keep-alive'}
req = request.Request(url, headers=headers)
page = request.urlopen(req).read()
page = page.decode('gbk')
page[:100]

import urllib.parse
url = r'https://new-api.meiqia.com/v1/throttle/web?api_name=%2Fvisit%2Finit&ent_id=7276&v=1534848690048'
headers = {'User-Agent': r'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) '
           r'Chrome/45.0.2454.85 Safari/537.36 115Browser/6.0.3',
           'Referer': r'http://www.lagou.com/zhaopin/Python/?labelWords=label',
           'Connection': 'keep-alive'}
values={'api_name':'/visit/init','ent_id':'7276','v':'1534848690048'}
data = urllib.parse.urlencode(values).encode(encoding='UTF8')
req = request.Request(url, data,headers=headers)
page = request.urlopen(req).read()
page = page.decode('gbk')

from urllib.request import Request, urlopen
from urllib.error import URLError, HTTPError
req = Request("http://www.dhdzp.com /")
try:
    response = urlopen(req)
except HTTPError as e:
    print('服務(wù)器無(wú)法滿(mǎn)足請(qǐng)求.')
    print('錯(cuò)誤代碼: ', e.code)
except URLError as e:
    print('不能訪問(wèn)服務(wù)器.')
    print('原因: ', e.reason)
else:
    print("OK!")
    print(response.read().decode("utf8"))

from urllib.request import Request, urlopen
from urllib.error import  URLError
req = Request("http://www.dhdzp.com /")
try:
    response = urlopen(req)
except URLError as e:
    if hasattr(e, '原因'):
        print('不能訪問(wèn)服務(wù)器.')
        print('原因: ', e.reason)
    elif hasattr(e, '代碼'):
        print('服務(wù)器無(wú)法滿(mǎn)足請(qǐng)求.')
        print('錯(cuò)誤代碼: ', e.code)
    else:
        print("OK!")
        print(response.read().decode("utf8"))

import urllib.request
 
# 私密代理授權(quán)的賬戶(hù)
user = "user_name"
# 私密代理授權(quán)的密碼
passwd = "uesr_password"
# 代理IP地址  比如可以使用百度西刺代理隨便選擇即可
proxyserver = "177.87.168.97:53281"
# 1. 構(gòu)建一個(gè)密碼管理對(duì)象，用來(lái)保存需要處理的用戶(hù)名和密碼
passwdmgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
# 2. 添加賬戶(hù)信息，第一個(gè)參數(shù)realm是與遠(yuǎn)程服務(wù)器相關(guān)的域信息，一般沒(méi)人管它都是寫(xiě)None，后面三個(gè)參數(shù)分別是 代理服務(wù)器、用戶(hù)名、密碼
passwdmgr.add_password(None, proxyserver, user, passwd)
# 3. 構(gòu)建一個(gè)代理基礎(chǔ)用戶(hù)名/密碼驗(yàn)證的ProxyBasicAuthHandler處理器對(duì)象，參數(shù)是創(chuàng)建的密碼管理對(duì)象
#   注意，這里不再使用普通ProxyHandler類(lèi)了
proxyauth_handler = urllib.request.ProxyBasicAuthHandler(passwdmgr)
# 4. 通過(guò) build_opener()方法使用這些代理Handler對(duì)象，創(chuàng)建自定義opener對(duì)象，參數(shù)包括構(gòu)建的 proxy_handler 和 proxyauth_handler
opener = urllib.request.build_opener(proxyauth_handler)
# 5. 構(gòu)造Request 請(qǐng)求
request = urllib.request.Request("http://bbs.pinggu.org/")
# 6. 使用自定義opener發(fā)送請(qǐng)求
response = opener.open(request)
# 7. 打印響應(yīng)內(nèi)容
print (response.read())

pip install BeautifulSoup

# 想要抓取我們需要的東西需要進(jìn)行定位，尋找到標(biāo)志
from bs4 import BeautifulSoup
soup = BeautifulSoup('<meta content="all" name="robots" />',"html.parser")
tag=soup.meta

# tag的類(lèi)別
type(tag)
>>> bs4.element.Tag

# tag的name屬性
tag.name
>>> 'meta'

# attributes屬性
tag.attrs
>>> {'content': 'all', 'name': 'robots'}

# BeautifulSoup屬性
type(soup)
>>> bs4.BeautifulSoup
soup.name
>>> '[document]'

# 字符串的提取
markup='<b><a  rel="external nofollow"  target="_blank">房產(chǎn)</a></b>'
soup=BeautifulSoup(markup,"lxml")
text=soup.b.string
text
>>> '房產(chǎn)'
type(text)
>>> bs4.element.NavigableString

import requests
from bs4 import BeautifulSoup
url = "http://www.cwestc.com/MroeNews.aspx?gd=2"
html = requests.get(url)
soup =  BeautifulSoup(html.text,"lxml")

#通過(guò)頁(yè)面解析得到結(jié)構(gòu)數(shù)據(jù)進(jìn)行處理
from bs4 import BeautifulSoup
soup=BeautifulSoup(html.text,"lxml")

#定位
lptable = soup.find('table',width='780')

# 解析
for i in lptable.find_all("td",width="680"):    
    title = i.b.strong.a.text
    ]
#     href = i.find('a')['href']
    date = href.split("/")[4]
    print (title,href,date)

from lxml import etree

html="""
<!DOCTYPE html>
<html>
<head lang="en">
    <title>test</title>
    <meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
</head>
<body>
<div id="content">
    <ul id="ul">
        <li>NO.1</li>
        <li>NO.2</li>
        <li>NO.3</li>
    </ul>
    <ul id="ul2">
        <li>one</li>
        <li>two</li>
    </ul>
</div>
<div id="url">
    <a  rel="external nofollow"  title="crossgate">crossgate</a>
    <a  rel="external nofollow"  title="pinggu">pinggu</a>
</div>
</body>
</html>
"""

#這里使用id屬性來(lái)定位哪個(gè)div和ul被匹配 使用text()獲取文本內(nèi)容
selector=etree.HTML(html)
content=selector.xpath('//div[@id="content"]/ul[@id="ul"]/li/text()') 
for i in content:
    print (i)

#這里使用//從全文中定位符合條件的a標(biāo)簽，使用“@標(biāo)簽屬性”獲取a便簽的href屬性值
con=selector.xpath('//a/@href') 
for i in con:
    print (i)

#使用絕對(duì)路徑 #使用相對(duì)路徑定位 兩者效果是一樣的
con=selector.xpath('/html/body/div/a/@title') 
print (len(con))
print (con[0],con[1])

import requests
from bs4 import BeautifulSoup
url = "http://www.cwestc.com/MroeNews.aspx?gd=1"
html = requests.get(url)
soup =  BeautifulSoup(html.text,"lxml")
soup.text[1900:2000]

import requests
from bs4 import BeautifulSoup
url = "http://www.cwestc.com/MroeNews.aspx?gd=2"
html = requests.get(url)
soup =  BeautifulSoup(html.text,"lxml")
soup.text[1900:2000]

import requests
from bs4 import BeautifulSoup
url = "http://news.cqcoal.com/blank/nl.jsp?tid=238"
html = requests.get(url)
soup =  BeautifulSoup(html.text,"lxml")
soup.text

import urllib
import urllib.request
import requests

url = "http://news.cqcoal.com/manage/newsaction.do?method:webListPageNewsArchivesByTypeid"
post_param = {'pageNum':'1',\
              'pageSize':'20',\
              'jsonStr':'{"typeid":"238"}'}
return_data = requests.post(url,data =post_param)
content=return_data.text
content