# 典型復(fù)雜字符串示例
log_line = "2023-05-01T08:15:23 | INFO | server-01 | User login | id:101, name:'Zhang San'"
csv_line = "ProductID:1024; Name:Python Cookbook; Price:45.99; Categories:Books,Programming"

# 基本split方法
parts = log_line.split('|')
# 結(jié)果: ['2023-05-01T08:15:23 ', ' INFO ', ' server-01 ', ' User login ', " id:101, name:'Zhang San'"]

import re

# 使用正則表達(dá)式拆分
address = "123 Main St, Suite 100, New York, NY 10001"
parts = re.split(r',|\s', address)  # 按逗號(hào)或空格拆分
# 結(jié)果: ['123', 'Main', 'St', '', 'Suite', '100', '', 'New', 'York', '', 'NY', '10001']

# 只拆分特定模式
text = "apple, banana; cherry: date"
parts = re.split(r'[,;:]', text)  # 匹配逗號(hào)、分號(hào)或冒號(hào)
# 結(jié)果: ['apple', ' banana', ' cherry', ' date']

# 使用捕獲分組保留分隔符
text = "Hello! How are you? I'm fine."
parts = re.split(r'([!?.])', text)  # 保留標(biāo)點(diǎn)符號(hào)
# 結(jié)果: ['Hello', '!', ' How are you', '?', " I'm fine", '.']

# 處理多種空格變體
text = "Python\tis\na great\r\nprogramming language"
parts = re.split(r'\s+', text)  # 匹配任意空白字符序列
# 結(jié)果: ['Python', 'is', 'a', 'great', 'programming', 'language']

def stateful_split(text, delimiters):
    """帶狀態(tài)的分割引擎，處理引號(hào)內(nèi)的分隔符"""
    tokens = []
    current = []
    in_quote = False
    quote_char = None
    
    for char in text:
        if char in ('"', "'") and not in_quote:
            in_quote = True
            quote_char = char
            current.append(char)
        elif char == quote_char and in_quote:
            in_quote = False
            quote_char = None
            current.append(char)
        elif char in delimiters and not in_quote:
            if current:
                tokens.append(''.join(current))
                current = []
        else:
            current.append(char)
    
    if current:
        tokens.append(''.join(current))
    
    return tokens

# 測(cè)試包含引號(hào)的字符串
text = 'name="Zhang, San" age=30 city="New, York"'
result = stateful_split(text, [' ', '=', ','])
# 結(jié)果: ['name', '"Zhang, San"', 'age', '30', 'city', '"New, York"']

def recursive_split(text, delimiters):
    """遞歸處理分層分隔符"""
    if not delimiters:
        return [text]
    
    current_delim = delimiters[0]
    remaining_delims = delimiters[1:]
    
    parts = []
    for part in text.split(current_delim):
        if remaining_delims:
            parts.extend(recursive_split(part, remaining_delims))
        else:
            parts.append(part)
    
    return parts

# 分層拆分示例
text = "A:B;C,D|E;F"
result = recursive_split(text, [';', ',', ':', '|'])
# 結(jié)果: ['A', 'B', 'C', 'D', 'E', 'F']

def stream_split(text, delimiters):
    """生成器實(shí)現(xiàn)流式分割，節(jié)省內(nèi)存"""
    current = []
    for char in text:
        if char in delimiters:
            if current:
                yield ''.join(current)
                current = []
        else:
            current.append(char)
    if current:
        yield ''.join(current)

# 處理大文件
with open('huge_file.txt') as f:
    for line in f:
        for token in stream_split(line, [',', ';', '|']):
            process_token(token)  # 流式處理每個(gè)token

def parse_log_line(line):
    """解析復(fù)雜日志格式"""
    # 定義日志格式: [時(shí)間] [級(jí)別] [服務(wù)器] [消息] [額外數(shù)據(jù)]
    pattern = r'\[(.*?)\] \[(.*?)\] \[(.*?)\] - (.*?) \| (.*)'
    match = re.match(pattern, line)
    
    if match:
        timestamp, level, server, message, extra = match.groups()
        
        # 解析額外數(shù)據(jù)
        extra_data = {}
        for item in re.split(r',\s*', extra):
            if ':' in item:
                key, value = re.split(r':\s*', item, 1)
                extra_data[key] = value.strip("'\"")
        
        return {
            'timestamp': timestamp,
            'level': level,
            'server': server,
            'message': message,
            'extra': extra_data
        }
    return None

# 示例日志
log_line = '[2023-05-01T08:15:23] [INFO] [server-01] - User login | id:101, name:"Zhang San", role:admin'
parsed = parse_log_line(log_line)

def clean_csv_line(line, delimiters=[',', ';', '|']):
    """處理混合分隔符的CSV行"""
    # 第一步：統(tǒng)一分隔符
    normalized = line
    for delim in delimiters[1:]:
        normalized = normalized.replace(delim, delimiters[0])
    
    # 第二步：處理引號(hào)內(nèi)的分隔符
    tokens = []
    current = []
    in_quote = False
    
    for char in normalized:
        if char == '"':
            in_quote = not in_quote
            current.append(char)
        elif char == delimiters[0] and not in_quote:
            tokens.append(''.join(current))
            current = []
        else:
            current.append(char)
    
    tokens.append(''.join(current))
    
    # 第三步：去除多余空格
    return [token.strip() for token in tokens]

# 測(cè)試混合分隔符CSV
csv_line = '101; "Zhang, San", 30; "New, York" | "Software Engineer"'
cleaned = clean_csv_line(csv_line)
# 結(jié)果: ['101', '"Zhang, San"', '30', '"New, York"', '"Software Engineer"']

def advanced_tokenizer(text):
    """高級(jí)文本分詞器"""
    # 處理縮寫和特殊符號(hào)
    text = re.sub(r"(\w+)'(\w+)", r"\1'\2", text)  # 保留I'm中的撇號(hào)
    text = re.sub(r"(\w+)\.(\w+)", r"\1.\2", text)  # 保留e.g.中的點(diǎn)
    
    # 定義分詞模式
    pattern = r'''
        \w+(?:-\w+)*        # 帶連字符的單詞
        | \d+\.\d+          # 浮點(diǎn)數(shù)
        | \d+               # 整數(shù)
        | \.\.\.            # 省略號(hào)
        | [^\w\s]           # 其他符號(hào)
    '''
    
    return re.findall(pattern, text, re.VERBOSE)

# 測(cè)試復(fù)雜文本
text = "I'm 99.9% sure that A.I. will change the world... don't you think?"
tokens = advanced_tokenizer(text)
# 結(jié)果: ["I'm", '99.9', '%', 'sure', 'that', 'A.I.', 'will', 'change', 'the', 'world', '...', "don't", 'you', 'think', '?']

# 預(yù)編譯常用模式
DELIMITER_PATTERN = re.compile(r'[,;:|]')
WHITESPACE_PATTERN = re.compile(r'\s+')

def optimized_split(text):
    """使用預(yù)編譯正則提高性能"""
    return DELIMITER_PATTERN.split(text)

# 性能對(duì)比（100萬次調(diào)用）：
# 未編譯: 2.8秒
# 預(yù)編譯: 1.2秒

# 使用Cython編寫高性能分割函數(shù)
# splitter.pyx
def cython_split(text, delimiters):
    cdef list tokens = []
    cdef list current = []
    cdef char c
    cdef set delim_set = set(delimiters)
    
    for c in text:
        if c in delim_set:
            if current:
                tokens.append(''.join(current))
                current = []
        else:
            current.append(c)
    
    if current:
        tokens.append(''.join(current))
    
    return tokens

# 編譯后調(diào)用
from splitter import cython_split
result = cython_split("a,b;c:d", [',', ';', ':'])

from concurrent.futures import ProcessPoolExecutor
import os

def parallel_file_split(file_path, delimiters, workers=4):
    """并行處理大文件分割"""
    results = []
    chunk_size = os.path.getsize(file_path) // workers
    
    with open(file_path, 'r') as f:
        with ProcessPoolExecutor(max_workers=workers) as executor:
            futures = []
            start = 0
            
            for i in range(workers):
                end = start + chunk_size
                if i == workers - 1:
                    end = None  # 最后一塊包含剩余內(nèi)容
                
                # 提交任務(wù)
                futures.append(executor.submit(
                    process_chunk, file_path, start, end, delimiters
                ))
                start += chunk_size
            
            # 收集結(jié)果
            for future in futures:
                results.extend(future.result())
    
    return results

def process_chunk(file_path, start, end, delimiters):
    """處理文件塊"""
    tokens = []
    with open(file_path, 'r') as f:
        if start > 0:
            f.seek(start)
            # 找到下一個(gè)完整行開始
            while f.read(1) not in ('\n', '\r'):
                start -= 1
                f.seek(start)
        
        # 讀取直到結(jié)束位置
        while True:
            pos = f.tell()
            if end is not None and pos >= end:
                break
                
            line = f.readline()
            if not line:
                break
                
            tokens.extend(advanced_split(line, delimiters))
    
    return tokens

# 空字符串處理
text = ",a,b,,c,"
# 錯(cuò)誤: ['', 'a', 'b', '', 'c', '']
# 正確: [x for x in text.split(',') if x] → ['a', 'b', 'c']

# 可讀性優(yōu)先
def parse_config_line(line):
    # 注釋處理
    if line.startswith('#') or not line.strip():
        return None
    
    # 鍵值分割
    if '=' in line:
        key, value = line.split('=', 1)
        return key.strip(), value.strip()
    
    return line.strip()

# 錯(cuò)誤：處理非ASCII分隔符
text = "日本＄東京＄中國＄北京"
parts = text.split('＄')  # 全角美元符號(hào)

# 解決方案：明確指定分隔符
delim = '＄'  # 直接使用實(shí)際字符

# 錯(cuò)誤：未轉(zhuǎn)義特殊字符
text = "a.b|c"
parts = re.split(r'.|', text)  # .和|在正則中有特殊含義

# 解決方案：正確轉(zhuǎn)義
parts = re.split(r'\.|\|', text)  # 結(jié)果: ['a', 'b', 'c']

# 危險(xiǎn)：一次性讀取大文件
with open('huge.log') as f:
    lines = f.readlines()  # 可能耗盡內(nèi)存
    for line in lines:
        parts = line.split('|')

# 解決方案：流式處理
with open('huge.log') as f:
    for line in f:
        parts = line.split('|')