from langchain.document_loaders import (
    UnstructuredHTMLLoader,
    BSHTMLLoader,
    WebBaseLoader,
    AsyncHtmlLoader
)

from langchain.document_loaders import BSHTMLLoader

loader = BSHTMLLoader("example.html")
data = loader.load()

print(f"文檔內(nèi)容：{data[0].page_content[:200]}...")
print(f"元數(shù)據(jù)：{data[0].metadata}")

from langchain.document_loaders import UnstructuredHTMLLoader

# 使用元素模式保留結(jié)構(gòu)
loader = UnstructuredHTMLLoader("blog_post.html", mode="elements")
docs = loader.load()

# 打印檢測(cè)到的元素類(lèi)型
for doc in docs:
    print(f"元素類(lèi)型: {doc.metadata['category']}")
    print(f"內(nèi)容: {doc.page_content[:80]}{'...' if len(doc.page_content) > 80 else ''}")
    print("-" * 50)

from langchain.document_loaders import WebBaseLoader

loader = WebBaseLoader([
    "https://example.com/dynamic-content",
    "https://web.with-js.com"
])
loader.requests_per_second = 2  # 禮貌爬取
docs = loader.load()

print(f"加載了{(lán)len(docs)}個(gè)文檔")

from langchain.document_loaders import AsyncHtmlLoader
from langchain.document_transformers import Html2TextTransformer

urls = [f"https://news-site.com/page/{i}" for i in range(1, 6)]
loader = AsyncHtmlLoader(urls)
html = loader.load()

# 轉(zhuǎn)換為純凈文本
html2text = Html2TextTransformer()
docs_transformed = html2text.transform_documents(html)

print(f"第一頁(yè)標(biāo)題：{docs_transformed[0].page_content.splitlines()[0]}")

# 錯(cuò)誤做法：默認(rèn)utf-8
loader = BSHTMLLoader("gbk_page.html")

# 正確做法：指定編碼
loader = BSHTMLLoader("gbk_page.html", encoding="gbk")

# 添加延遲和偽裝頭
loader = WebBaseLoader(
    urls,
    header_template={
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
        "Accept-Language": "en-US,en;q=0.9"
    },
    requests_per_second=2
)

# 確保啟用JS渲染
loader = WebBaseLoader("https://react-app.example")
loader.scrapejs = True  # 默認(rèn)已啟用

# 使用會(huì)話保持cookies
from requests.sessions import Session

session = Session()
session.post("https://site.com/login", data={"user": "...", "pass": "..."})

loader = WebBaseLoader("https://site.com/protected", session=session)

# 手動(dòng)處理滾動(dòng)
from playwright.sync_api import sync_playwright

with sync_playwright() as p:
    browser = p.chromium.launch()
    page = browser.new_page()
    page.goto("https://infinite-scroll.site")
    
    # 滾動(dòng)5次
    for _ in range(5):
        page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
        page.wait_for_timeout(2000)  # 等待加載
    
    content = page.content()
    loader = BSHTMLLoader.from_string(content)

# 使用代理輪詢
proxies = ["http://proxy1:port", "http://proxy2:port"]
loader = AsyncHtmlLoader(urls, proxies=proxies, rotate_proxy=True)

# 分批處理大文件
from langchain.text_splitter import RecursiveCharacterTextSplitter

splitter = RecursiveCharacterTextSplitter(
    chunk_size=2000,
    chunk_overlap=200
)

docs = loader.load()
chunks = splitter.split_documents(docs)

from langchain.document_loaders import WebBaseLoader
from langchain.document_transformers import Html2TextTransformer
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.embeddings import OpenAIEmbeddings
from langchain.vectorstores import Chroma

def html_processing_pipeline(url):
    # 1. 加載
    loader = WebBaseLoader(url)
    raw_docs = loader.load()
    
    # 2. 轉(zhuǎn)換
    html2text = Html2TextTransformer()
    cleaned_docs = html2text.transform_documents(raw_docs)
    
    # 3. 分塊
    splitter = RecursiveCharacterTextSplitter(
        chunk_size=1500,
        chunk_overlap=200,
        separators=["\n\n", "\n", "。", "！", "？"]
    )
    chunks = splitter.split_documents(cleaned_docs)
    
    # 4. 向量化
    embeddings = OpenAIEmbeddings()
    vectorstore = Chroma.from_documents(chunks, embeddings)
    
    return vectorstore

# 使用示例
store = html_processing_pipeline("https://example.com")
retriever = store.as_retriever(search_kwargs={"k": 3})

# 使用CSS選擇器精準(zhǔn)定位
from bs4 import BeautifulSoup

def extract_main_content(html):
    soup = BeautifulSoup(html, 'html.parser')
    
    # 策略1：嘗試常見(jiàn)內(nèi)容容器
    selectors = [
        'article', 
        '.post-content', 
        '.article-body',
        'main',
        '[role="main"]'
    ]
    
    for selector in selectors:
        element = soup.select_one(selector)
        if element and len(element.text) > 500:
            return element.get_text()
    
    # 策略2：回退到正文密度檢測(cè)
    all_text = soup.get_text()
    return all_text  # 實(shí)際應(yīng)用中應(yīng)實(shí)現(xiàn)正文密度算法

# 集成到LangChain
class SmartHTMLLoader(BSHTMLLoader):
    def load(self):
        raw_docs = super().load()
        for doc in raw_docs:
            doc.page_content = extract_main_content(doc.page_content)
        return raw_docs

from langchain.document_loaders import UnstructuredFileIOLoader

with open("large.html", "rb") as f:
    loader = UnstructuredFileIOLoader(f, strategy="fast")
    for chunk in loader.lazy_load():
        process(chunk)

from datetime import timedelta
from langchain.cache import SQLiteCache

loader = WebBaseLoader("https://news.com")
loader.cache = SQLiteCache(
    ttl=timedelta(hours=1),  # 1小時(shí)緩存
    db_path=".cache.db"
)

from langdetect import detect

docs = loader.load()
for doc in docs:
    lang = detect(doc.page_content)
    doc.metadata["language"] = lang
    if lang == "ja":  # 日語(yǔ)需要不同分句
        custom_split_japanese(doc)

# 結(jié)合截圖理解布局
from playwright.sync_api import sync_playwright

with sync_playwright() as p:
    browser = p.chromium.launch()
    page = browser.new_page()
    page.goto(url)
    page.screenshot(path="screenshot.png")
    
    # 使用CV模型分析布局
    layout = analyze_layout("screenshot.png")
    content = extract_by_coordinates(html, layout)

# 自動(dòng)化交互操作
loader = WebBaseLoader("https://dashboard.example")
loader.playwright_actions = [
    {"action": "click", "selector": "#load-more"},
    {"action": "type", "selector": "#search", "text": "keyword"},
    {"action": "wait_for", "selector": ".results"}
]