import requests

# **kwargs:控制訪問(wèn)的參數(shù)，均為可選項(xiàng)，不僅僅是針對(duì)request，其他六中方法依舊適用

# params：字典或字節(jié)序列，作為參數(shù)增加到URL中，可以通過(guò)該參數(shù)篩選數(shù)據(jù)
kv = {"key1":"value1","key2":"value2"}
r = requests.request('GET','http://python123.io/ws',params=kv)
print(r.url)
# https://python123.io//ws?key1=value1&key2=value2

# data：字典、字節(jié)序列或文件對(duì)象，作為Request的內(nèi)容;提交時(shí)，作為數(shù)據(jù)內(nèi)容添加到當(dāng)前的連接下
kv = {"key1":"value1","key2":"value2"}
r = requests.request('POST','http://python123.io/ws',params=kv)
body = '主體內(nèi)容'
r = requests.request('POST','http://python123.io/ws',params=body)

# json：JSON格式的數(shù)據(jù)，作為Request的內(nèi)容
kv = {"key1":"value1"}
r = requests.request('POST','http://python123.io/ws',json=kv)

# headers：字典，HTTP定制頭,模擬需要的瀏覽器來(lái)進(jìn)行訪問(wèn)
hd = {"user-agent":"Chrome/10"}
r = requests.request('POST','http://python123.io/ws',headers=hd)

# cookies：字典或CookieJar，Request中的cookie
# auth：元組，支持HTTP認(rèn)證功能
# files：字典類型，傳輸文件;將某個(gè)文件提交到連接上
fs = {"file":open('data.xls','rb')}
r = requests.request('POST','http://python123.io/ws',file=fs)

# timeout：設(shè)定超時(shí)時(shí)間，秒為單位;在規(guī)定的時(shí)間內(nèi)沒(méi)有接收到響應(yīng)將會(huì)顯示timeout異常
r = requests.request('POST','http://www.baidu.com',timeout=10)

# proxies：字典類型，設(shè)定訪問(wèn)代理服務(wù)器，可以增加登錄認(rèn)證
pxs = {'http':'http://user:pass@10.10.10.1:1234', #當(dāng)我們進(jìn)入HTTP協(xié)議的網(wǎng)站時(shí)增加登錄認(rèn)證
'https':'https://10.10.10.1.4321' } #當(dāng)我們進(jìn)入HTTPS協(xié)議的網(wǎng)站時(shí)，直接使用代理服務(wù)器的IP地址；可以有效掩蓋爬蟲(chóng)的原IP地址
r = requests.request('GET','http://python123.io/ws',proxies=pxs)

# allow_redirects：True/False，默認(rèn)為T(mén)rue，重定向開(kāi)關(guān)
# stream：True/False，默認(rèn)為T(mén)rue，獲取內(nèi)容立刻下載的開(kāi)關(guān)
# verify：True/False，默認(rèn)為T(mén)rue，認(rèn)證SSL證書(shū)開(kāi)關(guān)
# cert：本地SSL證書(shū)路徑

import requests

#構(gòu)造一個(gè)向服務(wù)器請(qǐng)求資源的Response對(duì)象
r = requests.get(url="http://www.baidu.com")

print(r.status_code) #打印請(qǐng)求狀態(tài)碼
#200
print(type(r)) #打印請(qǐng)求對(duì)象類型
#<class 'requests.models.Response'>
print(r.headers) #打印請(qǐng)求對(duì)象的頭部信息
#{'Cache-Control': 'private, no-cache, no-store, proxy-revalidate, no-transform', 'Connection': 'keep-alive', 'Content-Encoding': 'gzip', 'Content-Type': 'text/html', 'Date': 'Sat, 27 Jun 2020 09:03:41 GMT', 'Last-Modified': 'Mon, 23 Jan 2017 13:27:32 GMT', 'Pragma': 'no-cache', 'Server': 'bfe/1.0.8.18', 'Set-Cookie': 'BDORZ=27315; max-age=86400; domain=.baidu.com; path=/', 'Transfer-Encoding': 'chunked'}

print(r.text)
print(r.encoding) #ISO-8859-1
print(r.apparent_encoding) #備用編碼utf-8
r.encoding = "utf-8"
print(r.text)

try:
r = requests.get(url,timeout=30)
r.raise_for_status()
r.encoding = r.apparent_encoding
return r.text
except:
return "產(chǎn)生異常！"

import requests

# （定義方法）封裝函數(shù)
def getHTMLText(url):
try:
r = requests.get(url,timeout=30)
r.raise_for_status()
r.encoding = r.apparent_encoding
return r.text
except:
return "代碼錯(cuò)誤，產(chǎn)生異常！"

if __name__ =="__main__":
url = "http://www.baidu.com"
print(getHTMLText(url)) #正常顯示爬取的頁(yè)面信息

if __name__ =="__main__":
url = "www.baidu.com" #缺失了
print(getHTMLText(url)) #代碼錯(cuò)誤，產(chǎn)生異常！

import requests
# （定義方法）封裝函數(shù)
def getHTMLText(url):
try:
r = requests.get(url,timeout=30)
r.raise_for_status()
r.encoding = r.apparent_encoding
return r.text
except:
return "代碼錯(cuò)誤，產(chǎn)生異常！"

if __name__ =="__main__":
url = "http://www.baidu.com/robots.txt"
print(getHTMLText(url)) #正常顯示爬取的頁(yè)面信息,顯示出robots協(xié)議對(duì)于不同類型爬蟲(chóng)的限制