pip install requests-html

from requests_html import HTMLSession
session = HTMLSession()
response = session.get('https://example.com')
print(response.html.text)

# 使用選擇器提取標(biāo)題
titles = response.html.find('h2')
for title in titles:
    print(title.text)

# JavaScript渲染
r = session.get('https://example.com', params={'q': 'python'})
r.html.render()
print(r.html.text)

# 異步JavaScript渲染
r = session.get('https://example.com')
r.html.render(sleep=1, keep_page=True)
print(r.html.text)

# 自定義Headers和Cookies
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'}
cookies = {'example_cookie': 'value'}
r = session.get('https://example.com', headers=headers, cookies=cookies)
print(r.html.text)

# 抓取動態(tài)頁面
r = session.get('https://example.com/dynamic-page')
r.html.render()
print(r.html.text)

# 表單提交
payload = {'username': 'user', 'password': 'pass'}
r = session.post('https://example.com/login', data=payload)
print(r.html.text)

# 使用選擇器提取鏈接
links = response.html.find('a')
for link in links:
    print(link.attrs['href'])

# 使用更復(fù)雜的選擇器和過濾器
articles = response.html.find('article')
for article in articles:
    title = article.find('h2', first=True).text
    author = article.find('.author', first=True).text
    print(f"Title: {title}, Author: {author}")

# 等待頁面加載完成
r = session.get('https://example.com/dynamic-content')
r.html.render(wait=2)
print(r.html.text)

# 生成頁面截圖
r = session.get('https://example.com')
r.html.render(screenshot='screenshot.png')

# 異常處理和錯(cuò)誤頁面重試
try:
    r = session.get('https://example.com/unstable-page')
    r.html.render(retries=3, wait=2)
    print(r.html.text)
except Exception as e:
    print(f"Error: {e}")

from requests_html import AsyncHTMLSession
async def fetch(url):
    async with AsyncHTMLSession() as session:
        response = await session.get(url)
        return response.html.text
urls = ['https://example.com/page1', 'https://example.com/page2', 'https://example.com/page3']
# 利用asyncio.gather實(shí)現(xiàn)并發(fā)請求
results = AsyncHTMLSession().run(lambda: [fetch(url) for url in urls])
for result in results:
    print(result)

from requests_html import HTMLSession
session = HTMLSession()
# 利用連接池發(fā)送多個(gè)請求
responses = session.get(['https://example.com/page1', 'https://example.com/page2', 'https://example.com/page3'])
for response in responses:
    print(response.html.text)

from requests_html import HTMLSession
session = HTMLSession()
# 使用緩存
response = session.get('https://example.com', cached=True)
print(response.html.text)