C语言爬虫开发：常见错误与优化方案

// 错误示例 - 内存泄漏
void fetch_data() {
    char *buffer = malloc(1024);
    // 使用buffer获取数据
    // 忘记free(buffer)
}

// 正确做法 - 确保每个malloc都有对应的free
void fetch_data() {
    char *buffer = malloc(1024);
    if (buffer == NULL) {
        // 错误处理
        return;
    }
    
    // 使用buffer获取数据
    
    free(buffer); // 释放内存
}

// 错误示例 - 未检查socket连接是否成功
int sock = socket(AF_INET, SOCK_STREAM, 0);
connect(sock, (struct sockaddr *)&server, sizeof(server));
// 直接开始读写，没有检查连接是否成功

// 正确做法 - 检查每个系统调用的返回值
int sock = socket(AF_INET, SOCK_STREAM, 0);
if (sock < 0) {
    perror("socket创建失败");
    return -1;
}
​
// 设置超时
struct timeval timeout;
timeout.tv_sec = 10;
timeout.tv_usec = 0;
setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout));
​
if (connect(sock, (struct sockaddr *)&server, sizeof(server)) < 0) {
    perror("连接失败");
    close(sock);
    return -1;
}

// 错误示例 - 可能溢出
char path[100];
sprintf(path, "/api/data/%s", user_input);
// 如果user_input过长，会导致缓冲区溢出

// 正确做法 - 使用安全函数
char path[100];
snprintf(path, sizeof(path), "/api/data/%s", user_input);
​
// 或者动态分配
int needed = snprintf(NULL, 0, "/api/data/%s", user_input) + 1;
char *path = malloc(needed);
if (path) {
    snprintf(path, needed, "/api/data/%s", user_input);
    // 使用path...
    free(path);
}

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <unistd.h>
#include <netdb.h>
#include <ctype.h>
​
#define BUFFER_SIZE 4096
#define USER_AGENT "Mozilla/5.0 (compatible; MyCrawler/1.0)"
​
// 安全的字符串复制函数
void safe_strcpy(char *dest, const char *src, size_t dest_size) {
    strncpy(dest, src, dest_size - 1);
    dest[dest_size - 1] = '\0';
}
​
// 解析URL，提取主机名和路径
int parse_url(const char *url, char *host, size_t host_size, char *path, size_t path_size) {
    char *host_start = strstr(url, "://");
    if (host_start == NULL) {
        host_start = (char *)url;
    } else {
        host_start += 3;
    }
    
    char *path_start = strchr(host_start, '/');
    if (path_start != NULL) {
        size_t host_length = path_start - host_start;
        if (host_length >= host_size) return -1;
        safe_strcpy(host, host_start, min(host_length + 1, host_size));
        safe_strcpy(path, path_start, path_size);
    } else {
        safe_strcpy(host, host_start, host_size);
        safe_strcpy(path, "/", path_size);
    }
    
    return 0;
}
​
// 创建HTTP请求
char *build_http_request(const char *host, const char *path) {
    int size = snprintf(NULL, 0, 
        "GET %s HTTP/1.1\r\n"
        "Host: %s\r\n"
        "User-Agent: %s\r\n"
        "Connection: close\r\n"
        "\r\n", path, host, USER_AGENT);
    
    char *request = malloc(size + 1);
    if (request == NULL) return NULL;
    
    sprintf(request,
        "GET %s HTTP/1.1\r\n"
        "Host: %s\r\n"
        "User-Agent: %s\r\n"
        "Connection: close\r\n"
        "\r\n", path, host, USER_AGENT);
    
    return request;
}
​
// 提取HTML中的链接（简单示例）
void extract_links(const char *html, size_t html_len) {
    const char *ptr = html;
    while ((ptr = strstr(ptr, "href=\"")) != NULL) {
        ptr += 6; // 跳过href="
        const char *end = strchr(ptr, '"');
        if (end != NULL) {
            size_t len = end - ptr;
            char link[256];
            safe_strcpy(link, ptr, min(len + 1, sizeof(link)));
            printf("发现链接: %s\n", link);
            ptr = end;
        }
    }
}
​
// 主爬取函数
int crawl_url(const char *url) {
    char host[256] = {0};
    char path[256] = {0};
    
    if (parse_url(url, host, sizeof(host), path, sizeof(path)) != 0) {
        fprintf(stderr, "URL解析失败: %s\n", url);
        return -1;
    }
    
    // 解析主机名获取IP地址
    struct hostent *he = gethostbyname(host);
    if (he == NULL) {
        fprintf(stderr, "无法解析主机名: %s\n", host);
        return -1;
    }
    
    // 创建socket
    int sock = socket(AF_INET, SOCK_STREAM, 0);
    if (sock < 0) {
        perror("socket创建失败");
        return -1;
    }
    
    // 设置超时
    struct timeval timeout = {10, 0}; // 10秒超时
    setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout));
    
    // 连接服务器
    struct sockaddr_in server_addr;
    server_addr.sin_family = AF_INET;
    server_addr.sin_port = htons(80);
    server_addr.sin_addr = *((struct in_addr *)he->h_addr);
    
    if (connect(sock, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0) {
        perror("连接失败");
        close(sock);
        return -1;
    }
    
    // 构建并发送HTTP请求
    char *request = build_http_request(host, path);
    if (request == NULL) {
        fprintf(stderr, "请求构建失败\n");
        close(sock);
        return -1;
    }
    
    if (send(sock, request, strlen(request), 0) < 0) {
        perror("发送请求失败");
        free(request);
        close(sock);
        return -1;
    }
    free(request);
    
    // 接收响应
    char buffer[BUFFER_SIZE];
    ssize_t bytes_received;
    size_t total_received = 0;
    char *response = NULL;
    
    while ((bytes_received = recv(sock, buffer, sizeof(buffer) - 1, 0)) > 0) {
        buffer[bytes_received] = '\0';
        
        // 动态扩展响应缓冲区
        char *new_response = realloc(response, total_received + bytes_received + 1);
        if (new_response == NULL) {
            fprintf(stderr, "内存分配失败\n");
            free(response);
            close(sock);
            return -1;
        }
        response = new_response;
        memcpy(response + total_received, buffer, bytes_received);
        total_received += bytes_received;
        response[total_received] = '\0';
    }
    
    if (bytes_received < 0) {
        perror("接收数据失败");
        free(response);
        close(sock);
        return -1;
    }
    
    // 处理响应
    if (response != NULL) {
        // 查找HTTP响应头结束位置（\r\n\r\n）
        char *body = strstr(response, "\r\n\r\n");
        if (body != NULL) {
            body += 4; // 跳过空行
            printf("获取到 %zu 字节数据\n", total_received);
            extract_links(body, total_received - (body - response));
        }
        free(response);
    }
    
    close(sock);
    return 0;
}
​
int main() {
    const char *url = "http://example.com/page";
    printf("开始爬取: %s\n", url);
    
    if (crawl_url(url) == 0) {
        printf("爬取完成\n");
    } else {
        printf("爬取失败\n");
    }
    
    return 0;
}

gcc -o crawler crawler.c -lxml2