Python集合：高效处理无序唯一数据的利器

# 用户输入的标签列表（可能包含重复）
user_tags = ["python", "编程", "数据分析", "python", "机器学习", "编程"]
unique_tags = set(user_tags)
print(unique_tags)  # 输出: {'数据分析', 'python', '编程', '机器学习'}

# 检测用户权限（百万级数据测试）
import random
large_list = [random.randint(1, 10**6) for _ in range(10**6)]
large_set = set(large_list)
 
# 检测元素是否存在
target = 999999
%timeit target in large_list  # 约10ms
%timeit target in large_set  # 约50ns

# 尝试存储可变对象会报错
try:
    invalid_set = {[1, 2], [3, 4]}
except TypeError as e:
    print(e)  # 输出: unhashable type: 'list'

# 用户行为分析示例
user_a_actions = {"click", "scroll", "share", "like"}
user_b_actions = {"click", "comment", "share", "download"}
 
# 并集：所有不同行为
all_actions = user_a_actions | user_b_actions
print(all_actions)  # {'download', 'click', 'share', 'scroll', 'comment', 'like'}
 
# 交集：共同行为
common_actions = user_a_actions & user_b_actions
print(common_actions)  # {'click', 'share'}
 
# 差集：A有B没有的行为
a_unique_actions = user_a_actions - user_b_actions
print(a_unique_actions)  # {'scroll', 'like'}
 
# 对称差：不同时存在的行为
diff_actions = user_a_actions ^ user_b_actions
print(diff_actions)  # {'download', 'scroll', 'comment', 'like'}

# 找出两个列表中的共同元素（去重后）
list1 = [1, 2, 2, 3, 4, 4, 5]
list2 = [4, 5, 5, 6, 7, 8]
common_elements = {x for x in list1 if x in list2}
print(common_elements)  # {4, 5}
 
# 生成平方数集合（自动去重）
numbers = [1, 2, 2, 3, 3, 3]
squares = {x**2 for x in numbers}
print(squares)  # {1, 4, 9}

# 创建冻结集合
immutable_set = frozenset([1, 2, 3, 4])
 
# 作为字典键
graph = {
    frozenset([1, 2]): "edge1",
    frozenset([2, 3]): "edge2"
}
print(graph[frozenset([1, 2])])  # 输出: edge1

# 生成100万个随机数（约30%重复）
import random
data = [random.randint(1, 10**5) for _ in range(10**6)]
 
# 列表去重（慢）
def deduplicate_list(lst):
    seen = []
    for item in lst:
        if item not in seen:
            seen.append(item)
    return seen
 
# 集合去重（快）
def deduplicate_set(lst):
    return list(set(lst))
 
# 性能测试
%timeit deduplicate_list(data)  # 约1.2秒
%timeit deduplicate_set(data)  # 约80毫秒

# 统计日志中的唯一IP地址
log_entries = [
    "192.168.1.1 - GET /",
    "192.168.1.2 - POST /api",
    "192.168.1.1 - GET /home",
    "192.168.1.3 - GET /about",
    "192.168.1.1 - GET /contact"
]
 
# 提取IP并统计唯一值
ips = {entry.split()[0] for entry in log_entries}
print(f"Unique visitors: {len(ips)}")  # 输出: Unique visitors: 3

# 构建单词索引（倒排索引）
documents = [
    "python is great",
    "java is also great",
    "python and java are programming languages"
]
 
# 创建单词到文档ID的映射
index = {}
for doc_id, doc in enumerate(documents):
    for word in doc.split():
        if word not in index:
            index[word] = set()
        index[word].add(doc_id)
 
# 查询包含"python"和"great"的文档
query = {"python", "great"}
result_ids = set.intersection(*[index[word] for word in query if word in index])
print([documents[id] for id in result_ids])  # 输出: ['python is great']

# 错误示例
try:
    bad_set = {[1, 2], (3, 4)}
except TypeError as e:
    print(e)  # 输出: unhashable type: 'list'
 
# 正确做法：使用元组代替列表
good_set = {(1, 2), (3, 4)}

# 集合遍历顺序不确定
s = {1, 2, 3}
print([x for x in s])  # 可能输出 [1, 2, 3] 或 [3, 1, 2] 等
 
# 如果需要有序，可以排序后使用
ordered_list = sorted(s)
print(ordered_list)  # 始终输出 [1, 2, 3]

# 检查列表是否包含重复（小数据量）
data = [1, 2, 3, 4, 5]
 
# 方法1：使用集合（通用但稍慢）
has_duplicates = len(data) != len(set(data))
 
# 方法2：直接遍历（小数据更快）
has_duplicates = False
seen = set()
for item in data:
    if item in seen:
        has_duplicates = True
        break
    seen.add(item)
 
# 对于小数据量，方法2通常更快

# 简易布隆过滤器实现（仅演示概念）
import mmh3  # MurmurHash3算法
 
class SimpleBloomFilter:
    def __init__(self, size=1000):
        self.size = size
        self.bits = [False] * size
    
    def add(self, item):
        # 使用两个哈希函数
        hash1 = mmh3.hash(str(item), 0) % self.size
        hash2 = mmh3.hash(str(item), 42) % self.size
        self.bits[hash1] = True
        self.bits[hash2] = True
    
    def __contains__(self, item):
        hash1 = mmh3.hash(str(item), 0) % self.size
        hash2 = mmh3.hash(str(item), 42) % self.size
        return self.bits[hash1] and self.bits[hash2]
 
# 使用示例
bf = SimpleBloomFilter()
words = ["apple", "banana", "cherry"]
for word in words:
    bf.add(word)
 
print("apple" in bf)    # True
print("orange" in bf)   # False（可能误判）

# 处理大型日志文件（模拟）
def generate_log_entries(file_path):
    with open(file_path) as f:
        for line in f:
            yield line.split()[0]  # 假设第一列是IP
 
# 统计唯一IP（不加载整个文件到内存）
def count_unique_ips(file_path):
    ip_set = set()
    for ip in generate_log_entries(file_path):
        ip_set.add(ip)
    return len(ip_set)
 
# 实际使用时，可以这样调用
# unique_count = count_unique_ips("access.log")

import json
 
# 集合序列化
user_permissions = {"read", "write", "execute"}
 
# 直接序列化会报错（因为集合不可JSON序列化）
try:
    json.dumps(user_permissions)
except TypeError as e:
    print(e)  # 输出: Object of type set is not JSON serializable
 
# 解决方案1：转换为列表
json_data = json.dumps(list(user_permissions))
print(json_data)  # 输出: ["read", "write", "execute"]
 
# 解决方案2：自定义编码器
class SetEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, set):
            return list(obj)
        return super().default(obj)
 
json_data = json.dumps(user_permissions, cls=SetEncoder)
print(json_data)  # 输出: ["read", "write", "execute"]