# 基礎(chǔ)strip方法
text = "  Hello World! \t\n"
clean_text = text.strip()  # "Hello World!"

# 指定移除字符
filename = "$$$report.txt$$$"
clean_file = filename.strip('$')  # "report.txt"

# 左右分別處理
text = "===[Important]==="
clean_left = text.lstrip('=')  # "[Important]==="
clean_right = text.rstrip('=')  # "===[Important]"

# 基礎(chǔ)replace
text = "Python\tis\nawesome"
clean_text = text.replace('\t', ' ').replace('\n', ' ')  # "Python is awesome"

# 多字符批量替換
def multi_replace(text, replacements):
    for old, new in replacements.items():
        text = text.replace(old, new)
    return text

replace_map = {'\t': ' ', '\n': ' ', '\r': ''}
clean_text = multi_replace(text, replace_map)

import re

# 移除所有非字母數(shù)字字符
text = "Product#123 costs $99.99!"
clean_text = re.sub(r'[^\w\s]', '', text)  # "Product123 costs 9999"

# 保留特定字符集
def keep_specific_chars(text, allowed):
    pattern = f"[^{re.escape(allowed)}]"
    return re.sub(pattern, '', text)

# 只保留中文和數(shù)字
clean_text = keep_specific_chars("中文ABC123", "\u4e00-\u9fa50-9")  # "中文123"

# 移除HTML標簽
html = "<div>Hello <b>World</b></div>"
clean_text = re.sub(r'<[^>]+>', '', html)  # "Hello World"

# 移除XML/HTML注釋
xml = "<!-- Header --><content>Text</content><!-- Footer -->"
clean_xml = re.sub(r'<!--.*?-->', '', xml, flags=re.DOTALL)  # "<content>Text</content>"

# 移除控制字符
def remove_control_chars(text):
    # 移除ASCII控制字符 (0-31和127)
    text = re.sub(r'[\x00-\x1F\x7F]', '', text)
    # 移除Unicode控制字符
    return re.sub(r'\p{C}', '', text, flags=re.UNICODE)

# 創(chuàng)建轉(zhuǎn)換表
trans_table = str.maketrans('', '', '!@#$%^&*()_+')

text = "Clean_this!@string"
clean_text = text.translate(trans_table)  # "Cleanthisstring"

# 復雜映射：替換和刪除組合
trans_map = str.maketrans({
    '\t': ' ',      # 制表符替換為空格
    '\n': ' ',      # 換行符替換為空格
    '\r': None,     # 回車符刪除
    '\u2028': None  # 行分隔符刪除
})
clean_text = text.translate(trans_map)

import unicodedata

def remove_diacritics(text):
    """移除變音符號"""
    # 分解字符
    nfd_text = unicodedata.normalize('NFD', text)
    # 移除非間距標記
    return ''.join(
        c for c in nfd_text 
        if unicodedata.category(c) != 'Mn'
    )

# 示例
text = "Café na?ve fa?ade"
clean_text = remove_diacritics(text)  # "Cafe naive facade"

# 全角轉(zhuǎn)半角
def full_to_half(text):
    """全角字符轉(zhuǎn)半角"""
    trans_map = {}
    for char in text:
        unicode_name = unicodedata.name(char, '')
        if 'FULLWIDTH' in unicode_name:
            half_char = chr(ord(char) - 0xFEE0)
            trans_map[char] = half_char
    return text.translate(str.maketrans(trans_map))

# 示例
text = "ＡＢＣ１２３"
clean_text = full_to_half(text)  # "ABC123"

def clean_currency(text):
    """凈化貨幣字符串"""
    # 步驟1: 移除非數(shù)字和分隔符
    text = re.sub(r'[^\d.,-]', '', text)
    
    # 步驟2: 統(tǒng)一千位分隔符
    text = text.replace(',', '')
    
    # 步驟3: 小數(shù)位處理
    if '.' in text and ',' in text:
        # 確定小數(shù)分隔符（最后一個分隔符）
        if text.rfind('.') > text.rfind(','):
            text = text.replace(',', '')
        else:
            text = text.replace('.', '').replace(',', '.')
    
    # 步驟4: 轉(zhuǎn)換為浮點數(shù)
    try:
        return float(text)
    except ValueError:
        return None

# 測試
currencies = [
    "$1,234.56",       # 標準美元
    "1.234,56 €",      # 歐洲格式
    "JPY 123,456",     # 日元
    "RMB 9.876,54"     # 人民幣
]

cleaned = [clean_currency(c) for c in currencies]
# [1234.56, 1234.56, 123456.0, 9876.54]

def clean_stock_code(code):
    """凈化證券代碼"""
    # 1. 移除所有非字母數(shù)字字符
    code = re.sub(r'[^\w]', '', code)
    
    # 2. 統(tǒng)一大小寫
    code = code.upper()
    
    # 3. 識別交易所前綴
    exchange_map = {
        'SH': 'SS',    # 上海
        'SZ': 'SZ',    # 深圳
        'HK': 'HK',    # 香港
        'US': ''       # 美國無前綴
    }
    
    # 4. 處理前綴
    for prefix, replacement in exchange_map.items():
        if code.startswith(prefix):
            code = replacement + code[len(prefix):]
            break
    
    return code

# 測試
codes = ["SH600000", "sz000001", " us_aapl ", "HK.00700"]
cleaned = [clean_stock_code(c) for c in codes]
# ['SS600000', 'SZ000001', 'AAPL', 'HK00700']

def anonymize_log(log_line):
    """日志敏感信息脫敏"""
    # 手機號脫敏
    log_line = re.sub(r'(\d{3})\d{4}(\d{4})', r'\1****\2', log_line)
    
    # 身份證脫敏
    log_line = re.sub(r'(\d{4})\d{10}(\w{4})', r'\1**********\2', log_line)
    
    # 郵箱脫敏
    log_line = re.sub(
        r'([a-zA-Z0-9._%+-]+)@([a-zA-Z0-9.-]+\.[a-zA-Z]{2,})', 
        r'***@\2', 
        log_line
    )
    
    # IP地址脫敏
    log_line = re.sub(
        r'\b(\d{1,3})\.(\d{1,3})\.\d{1,3}\.\d{1,3}\b', 
        r'\1.\2.***.***', 
        log_line
    )
    
    return log_line

# 示例日志
log = "User: john@example.com, IP: 192.168.1.100, Phone: 13800138000, ID: 510106199001011234"
safe_log = anonymize_log(log)
# "User: ***@example.com, IP: 192.168.***.***, Phone: 138****8000, ID: 5101**********1234"

class LogCleaner:
    """大日志文件流式清洗器"""
    def __init__(self, clean_functions):
        self.clean_functions = clean_functions
        self.buffer = ""
        self.chunk_size = 4096
    
    def clean_stream(self, input_stream, output_stream):
        """流式清洗處理"""
        while True:
            chunk = input_stream.read(self.chunk_size)
            if not chunk:
                break
            
            self.buffer += chunk
            while '\n' in self.buffer:
                line, self.buffer = self.buffer.split('\n', 1)
                cleaned = self.clean_line(line)
                output_stream.write(cleaned + '\n')
        
        # 處理剩余內(nèi)容
        if self.buffer:
            cleaned = self.clean_line(self.buffer)
            output_stream.write(cleaned)
    
    def clean_line(self, line):
        """單行清洗處理"""
        for clean_func in self.clean_functions:
            line = clean_func(line)
        return line

# 使用示例
def remove_timestamps(line):
    return re.sub(r'\[\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\]', '', line)

def remove_debug(line):
    return re.sub(r'DEBUG:.*?;', '', line)

cleaner = LogCleaner([remove_timestamps, remove_debug])

with open('large_app.log', 'r') as fin, open('clean_app.log', 'w') as fout:
    cleaner.clean_stream(fin, fout)

def unify_unicode(text):
    """統(tǒng)一Unicode字符表示"""
    # 步驟1: 兼容性規(guī)范化
    text = unicodedata.normalize('NFKC', text)
    
    # 步驟2: 處理特殊空白字符
    whitespace_map = {
        '\u00A0': ' ',   # 不換行空格
        '\u200B': '',    # 零寬空格
        '\u200C': '',    # 零寬非連接符
        '\u200D': '',    # 零寬連接符
        '\uFEFF': ''     # 字節(jié)順序標記
    }
    text = text.translate(str.maketrans(whitespace_map))
    
    # 步驟3: 替換易混淆字符
    confusables_map = {
        '０': '0', '１': '1', '２': '2', # 全角數(shù)字
        'Ａ': 'A', 'Ｂ': 'B', 'Ｃ': 'C', # 全角字母
        '。': '.', '，': ',', '；': ';'  # 全角標點
    }
    return text.translate(str.maketrans(confusables_map))

# 測試
mixed_text = "Ｈｅｌｌｏ?。祝铮颍欤洌?
clean_text = unify_unicode(mixed_text)  # "Hello World!"

def handle_emojis(text, mode='remove'):
    """表情符號處理"""
    # Unicode表情符號范圍
    emoji_pattern = re.compile(
        r'[\U0001F600-\U0001F64F'  # 表情符號
        r'\U0001F300-\U0001F5FF'   # 其他符號和象形文字
        r'\U0001F680-\U0001F6FF'   # 交通和地圖符號
        r'\U0001F700-\U0001F77F'   # 煉金術(shù)符號
        r']', 
        flags=re.UNICODE
    )
    
    if mode == 'remove':
        return emoji_pattern.sub('', text)
    elif mode == 'replace':
        return emoji_pattern.sub('[EMOJI]', text)
    elif mode == 'extract':
        return emoji_pattern.findall(text)
    else:
        return text

# 示例
text = "Python is awesome! ????"
print(handle_emojis(text, 'remove'))   # "Python is awesome! "
print(handle_emojis(text, 'replace'))  # "Python is awesome! [EMOJI][EMOJI]"
print(handle_emojis(text, 'extract'))   # ['??', '??']

import timeit

# 測試數(shù)據(jù)
text = "a" * 10000 + "!@#$%" + "b" * 10000

# 測試函數(shù)
def test_strip():
    return text.strip('!@#$%')

def test_replace():
    return text.replace('!', '').replace('@', '').replace('#', '').replace('$', '').replace('%', '')

def test_re_sub():
    return re.sub(r'[!@#$%]', '', text)

def test_translate():
    trans = str.maketrans('', '', '!@#$%')
    return text.translate(trans)

# 性能測試
methods = {
    "strip": test_strip,
    "replace": test_replace,
    "re_sub": test_re_sub,
    "translate": test_translate
}

results = {}
for name, func in methods.items():
    time = timeit.timeit(func, number=1000)
    results[name] = time

# 打印結(jié)果
for name, time in sorted(results.items(), key=lambda x: x[1]):
    print(f"{name}: {time:.4f}秒")

# 高性能字符移除
trans_table = str.maketrans('', '', '!@#$%')
clean_text = text.translate(trans_table)

# 預編譯正則對象
pattern = re.compile(r'[\W]')
clean_text = pattern.sub('', text)

# 分塊處理避免內(nèi)存溢出
with open('huge.txt') as f:
    while chunk := f.read(4096):
        process(chunk)

def clean_pipeline(text):
    text = remove_control_chars(text)
    text = unify_whitespace(text)
    text = normalize_unicode(text)
    return text

def context_aware_clean(text):
    if is_financial(text):
        return clean_currency(text)
    elif is_log_entry(text):
        return anonymize_log(text)
    else:
        return basic_clean(text)

import unittest

class TestCleaning(unittest.TestCase):
    def test_currency_cleaning(self):
        self.assertEqual(clean_currency("$1,000.50"), 1000.5)
        self.assertEqual(clean_currency("1.000,50€"), 1000.5)
    
    def test_log_anonymization(self):
        original = "User: john@example.com"
        expected = "User: ***@example.com"
        self.assertEqual(anonymize_log(original), expected)