pip install pyquery requests

import requests
from pyquery import PyQuery as pq

url = "https://example.com"
response = requests.get(url)
html = response.text # 獲取響應(yīng)文本

doc = pq(html)  # 直接傳入HTML字符串
# 或從文件加載
# with open("page.html") as f:
#     doc = pq(f.read())

# 獲取所有<a>標(biāo)簽
links = doc("a")

# 獲取class為"title"的元素
titles = doc(".title")

# 獲取id為"main"的元素
main = doc("#main")

# 組合選擇器
items = doc("div.product > h2")

<div class="product-item">
    <h2 class="name">商品名稱</h2>
    <span class="price">￥99.99</span>
    <a href="/product/123" rel="external nofollow"  class="detail-link">查看詳情</a>
</div>

def extract_products(url):
    response = requests.get(url)
    doc = pq(response.text)
    
    products = []
    items = doc(".product-item")  # 選擇所有商品項
    
    for item in items.items():
        name = item(".name").text()  # 提取名稱
        price = item(".price").text()  # 提取價格
        link = item(".detail-link").attr("href")  # 提取鏈接
        
        products.append({
            "name": name,
            "price": price,
            "link": link
        })
    
    return products

products = extract_products("https://shop.example.com")
for p in products[:2]:  # 打印前兩個商品
    print(f"商品: {p['name']}, 價格: {p['price']}, 鏈接: {p['link']}")

# 獲取所有商品名稱并去重
names = doc(".product-item .name").text().split()
unique_names = list(set(names))

from selenium import webdriver


driver = webdriver.Chrome()
driver.get("https://dynamic.example.com")
html = driver.page_source
doc = pq(html)
# 后續(xù)解析同上

first_product = doc(".product-item:first")
last_price = doc(".price:last").text()

# 遍歷所有商品
for item in doc(".product-item").items():
    print(item(".name").text())

# 過濾價格大于100的商品
expensive = [p for p in products if float(p["price"][1:]) > 100]

response.encoding = "utf-8" # 或 "gbk"

headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36..."
}
response = requests.get(url, headers=headers)

price = item(".price").text() if item(".price") else "N/A"

def get_headlines(url):
    response = requests.get(url, headers={"User-Agent": "Mozilla/5.0"})
    doc = pq(response.text)
    
    headlines = []
    for item in doc(".headline").items():
        title = item("h1").text()
        link = item("a").attr("href")
        if title and link:  # 確保元素存在
            headlines.append({
                "title": title,
                "link": link
            })
    
    return headlines

news = get_headlines("https://news.example.com")
import json
with open("headlines.json", "w", encoding="utf-8") as f:
    json.dump(news, f, ensure_ascii=False, indent=2)

# 不推薦：全局查找
# titles = doc(".title").text()


# 推薦：先定位容器
container = doc("#main-content")
titles = container(".title").text()

from lxml import etree
root = etree.HTML(html)
prices = root.xpath('//span[@class="price"]/text()')

proxies = {
    "http": "http://123.123.123.123:8080",
    "https": "https://123.123.123.123:8080"
}
response = requests.get(url, proxies=proxies)

session = requests.Session()
login_data = {"username": "user", "password": "pass"}
session.post("https://example.com/login", data=login_data)
response = session.get("https://example.com/dashboard")

# 方法1：從響應(yīng)頭獲取編碼
response.encoding = response.apparent_encoding


# 方法2：手動指定
doc = pq(response.text.encode("latin1").decode("gbk"))

# 假設(shè)"加載更多"觸發(fā)的是以下API
api_url = "https://example.com/api/products?page=2"
data = requests.get(api_url).json() # 獲取JSON數(shù)據(jù)