基于C语言实现网络爬虫程序设计

# Ubuntu/Debiansudo apt-get install libcurl4-openssl-dev libxml2-dev

#include <stdio.h>#include <stdlib.h>#include <string.h>#include <curl/curl.h>#include <libxml/HTMLparser.h>#include <libxml/xpath.h>
// 存储HTTP响应内容的结构体struct MemoryStruct {    char *memory;    size_t size;};
// libcurl回调函数：将响应内容写入内存static size_t WriteMemoryCallback(void *contents, size_t size, size_t nmemb, void *userp) {    size_t realsize = size * nmemb;    struct MemoryStruct *mem = (struct MemoryStruct *)userp;
    mem->memory = realloc(mem->memory, mem->size + realsize + 1);    if (mem->memory == NULL) {        printf("内存分配失败！\n");        return 0;    }
    memcpy(&(mem->memory[mem->size]), contents, realsize);    mem->size += realsize;    mem->memory[mem->size] = 0; // Null-terminate
    return realsize;}
// 从HTML中提取所有链接void extract_links(const char *html_content, const char *base_url) {    xmlDoc *doc = htmlReadDoc((const xmlChar*)html_content, base_url, NULL, HTML_PARSE_RECOVER | HTML_PARSE_NOERROR | HTML_PARSE_NOWARNING);    if (doc == NULL) return;
    xmlXPathContext *context = xmlXPathNewContext(doc);    if (context == NULL) {        xmlFreeDoc(doc);        return;    }
    // XPath表达式查找所有<a>标签的href属性    xmlXPathObject *result = xmlXPathEvalExpression((const xmlChar*)"//a/@href", context);    if (result == NULL) {        xmlXPathFreeContext(context);        xmlFreeDoc(doc);        return;    }
    // 遍历结果并打印链接    xmlNodeSet *nodeset = result->nodesetval;    for (int i = 0; i < nodeset->nodeNr; i++) {        xmlChar *href = xmlNodeListGetString(doc, nodeset->nodeTab[i]->children, 1);        printf("发现链接: %s\n", href);        xmlFree(href);    }
    // 清理资源    xmlXPathFreeObject(result);    xmlXPathFreeContext(context);    xmlFreeDoc(doc);}
// 抓取指定URL的页面内容void crawl_url(const char *url) {    CURL *curl_handle;    CURLcode res;    struct MemoryStruct chunk;
    chunk.memory = malloc(1); // 初始化为空    chunk.size = 0;
    curl_global_init(CURL_GLOBAL_ALL);    curl_handle = curl_easy_init();
    // 设置CURL选项    curl_easy_setopt(curl_handle, CURLOPT_URL, url);    curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);    curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, (void *)&chunk);    curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, "Mozilla/5.0 (兼容性爬虫)");    curl_easy_setopt(curl_handle, CURLOPT_FOLLOWLOCATION, 1L); // 跟随重定向
    // 执行请求    res = curl_easy_perform(curl_handle);
    if (res != CURLE_OK) {        fprintf(stderr, "请求失败: %s\n", curl_easy_strerror(res));    } else {        printf("成功抓取: %s (%zu字节)\n", url, chunk.size);        extract_links(chunk.memory, url); // 解析链接    }
    // 清理资源    curl_easy_cleanup(curl_handle);    free(chunk.memory);    curl_global_cleanup();}
int main() {    const char *start_url = "Example Domain"; // 起始URL    crawl_url(start_url);    return 0;}