4 万行客户表去重：精确匹配只清掉 37%，剩下全靠三层模糊｜PRIVO AI智能操作助手 三合一·网页+桌面+IDE

import re

SUFFIX = r"(有限责任公司|股份有限公司|集团有限公司|有限公司|公司)$"
BRACKET = r"[（(\[【][^）)\]】]*[）)\]】]"

def normalize(name: str) -> str:
    s = name.strip().replace("　", "")
    s = s.translate(str.maketrans("（）【】", "()[]"))
    s = re.sub(BRACKET, "", s)
    s = re.sub(r"\s+", "", s)
    return re.sub(SUFFIX, "", s)

from difflib import SequenceMatcher
from collections import defaultdict

def bucket_key(s: str) -> str:
    return s[:2] if len(s) >= 2 else s

buckets = defaultdict(list)
for idx, row in enumerate(rows):
    buckets[bucket_key(row["norm"])].append(idx)

pairs = []
for idxs in buckets.values():
    for i in range(len(idxs)):
        for j in range(i + 1, len(idxs)):
            a, b = rows[idxs[i]]["norm"], rows[idxs[j]]["norm"]
            if abs(len(a) - len(b)) > 3:
                continue
            r = SequenceMatcher(None, a, b).ratio()
            if r >= 0.88:
                pairs.append((idxs[i], idxs[j], round(r, 3)))

from pypinyin import lazy_pinyin

def initials(s: str) -> str:
    return "".join(p[0] for p in lazy_pinyin(s) if p)

for idxs in buckets.values():
    sig = defaultdict(list)
    for i in idxs:
        sig[initials(rows[i]["norm"])].append(i)
    for k, group in sig.items():
        if len(group) > 1:
            print("待确认:", k, [rows[i]["name"] for i in group])