python解析web访问日志

127.0.0.1 - - [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326

import re

# 定义日志文件的路径
log_file_path = 'path_to_your_log_file.log'

# 正则表达式匹配日志行的模式
log_pattern = re.compile(
    r'(?P<ip>\d+\.\d+\.\d+\.\d+) - - '
    r'\[(?P<datetime>[^\]]+)\] '
    r'"(?P<method>\w+) (?P<path>[^\s]+) (?P<protocol>[^"]+)" '
    r'(?P<status>\d+) (?P<size>\d+)'
)

# 解析日志文件
def parse_log_file(path):
    with open(path, 'r') as file:
        for line in file:
            match = log_pattern.match(line)
            if match:
                data = match.groupdict()
                print(data)

# 调用函数
parse_log_file(log_file_path)

import pandas as pd
import re

# 正则表达式匹配日志行的模式
log_pattern = re.compile(
    r'(?P<ip>\d+\.\d+\.\d+\.\d+) - - '
    r'\[(?P<datetime>[^\]]+)\] '
    r'"(?P<method>\w+) (?P<path>[^\s]+) (?P<protocol>[^"]+)" '
    r'(?P<status>\d+) (?P<size>\d+)'
)

# 解析日志文件并返回DataFrame
def parse_log_to_df(path):
    records = []
    with open(path, 'r') as file:
        for line in file:
            match = log_pattern.match(line)
            if match:
                records.append(match.groupdict())
    return pd.DataFrame(records)

# 转换日期时间格式并设置为索引
def process_data(df):
    df['datetime'] = pd.to_datetime(df['datetime'], format='%d/%b/%Y:%H:%M:%S %z')
    df.set_index('datetime', inplace=True)
    return df

# 读取日志文件
log_df = parse_log_to_df(log_file_path)

# 处理数据
log_df = process_data(log_df)

# 示例分析：计算每小时的访问量
hourly_traffic = log_df.resample('H').size()
print(hourly_traffic)

# 示例分析：统计状态码的出现次数
status_counts = log_df['status'].value_counts()
print(status_counts)