批量采集列表，快速定位xpath2024.4.9

用户7138673

发布于 2024-04-10 19:00:47

970

发布于 2024-04-10 19:00:47

文章被收录于专栏：大大的小数据

1、做了一个程序，输入网址和xpath就可以批量采集列表标题和标题链接

# -*- coding: utf-8 -*-
"""
Created on Thu Jan 28 10:08:38 2021

@author: Administrator
"""
# from DrissionPage import MixPage
from DrissionPage import WebPage
from DrissionPage import SessionPage
import re
from time import sleep
import csv
import os, sys
import datetime
import pysnooper
from time import sleep
from tenacity import retry, stop_after_attempt, wait_fixed, stop_after_delay
import pandas
from DataRecorder import Recorder  # 记录器
from openpyxl import load_workbook

#now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
now = datetime.datetime.now()
today = datetime.date.today()
yesterday = str(today - datetime.timedelta(days=1))
the_day_before_yesterday = str(today - datetime.timedelta(days=2))
today_4 = str(today - datetime.timedelta(days=3))

#@retry(stop=stop_after_attempt(2),wait=wait_fixed(2))
#@pysnooper.snoop(normalize= True,prefix="主程序")
def main无头浏览器():
#获取主页
    # p = MixPage('s')
    # p = WebPage()
    p = SessionPage()
    count = 0
#    r = Recorder('批量爬虫2021.5.24.xlsx', 50)  # 50表示每50条记录写入一次文件
    #打开热榜CSV
#    with open(r'C:\Users\Administrator\Desktop\python\关键字抓取2021.3.30.csv', 'r', newline = '\n')as g:
#        for row in csv.reader(g):
                #            print(row[9])
                
    关键字抓取网址表 = pandas.read_excel(r'C:\Users\Administrator\Desktop\python\关键字抓取2024.4.10-1.xlsx',sheet_name='Sheet1',header=None) #, nrows=50
    wb = load_workbook('批量爬虫2021.5.24.xlsx')
    sheet = wb.active
    for row in 关键字抓取网址表.values:
#        wb = load_workbook('批量爬虫2021.5.24.xlsx')
#        sheet = wb.active
#        print(row[9])
    #    print(行[0])                
#        count = 0
        if row[9] == "p = MixPage('s')" :
            count +=1
            print(count)
            print(row[9])
            原网站名 = row[6]
            主页链接 = row[7]
            定位xpath = row[10]
#            r = Recorder('批量爬虫2021.5.24.xlsx', 50)  # 50表示每50条记录写入一次文件
            try:
#                '''
                p.get(主页链接) 
                #p.driver.minimize_window()
                网站名 = p.title
                一级网址 = p.url
#                print(网站名)
#                print(一级网址)
        #        p.to_iframe('iframe')
#                x = '//*[@id="lab-show"]/div[2]/div[2]/div/ul/li/a'
                datas = p.eles('xpath:'+定位xpath)
#                r = Recorder('批量爬虫2021.5.24.xlsx', 1)  # 50表示每50条记录写入一次文件
                for data in datas:
                    内容标题 = data.text.replace(' ', '').replace('\n', '').replace('\r', '')
                    内容网址 = data.link
                    sheet.append((now,today,网站名,一级网址,内容标题,内容网址))
#                    r = Recorder('批量爬虫2021.5.24.xlsx', 1)  # 50表示每50条记录写入一次文件
#                    r.add_data((now,today,网站名,一级网址,内容标题,内容网址))  # 插入一条数据（也可一次插入多条）
#                    print(内容标题)
#                    print(内容网址)
        
                    #把新链接写入热榜CSV文件内
        #            with open('C:\\Users\\Administrator\\Desktop\\批量爬虫2021.5.24.csv', 'a+', newline = '\n',encoding='utf-8')as f:
#                    with open(r'C:\Users\Administrator\Desktop\python\批量爬虫2021.5.24.csv', 'a+', newline = '\n')as f:
#                        w = csv.writer(f)
#                        w.writerow([now,today,网站名,一级网址,内容标题,内容网址])
            except:
                print('出错跳过了'+row[6])
                with open(r'C:\Users\Administrator\Desktop\python\批量爬虫失败日志2021.6.11.csv', 'a+', newline = '\n')as f:
                        w = csv.writer(f)
                        w.writerow([now,today,原网站名,主页链接])
        wb.save('批量爬虫2021.5.24.xlsx')
        # p.close_driver()
#            '''
main无头浏览器()





#@retry(stop=stop_after_attempt(2),wait=wait_fixed(2))
def main谷歌浏览器():
#获取主页
    # p = MixPage()
    p = WebPage()

    count = 0
#    r = Recorder('批量爬虫2021.5.24.xlsx', 50)  # 50表示每50条记录写入一次文件
    #打开热榜CSV
#    with open(r'C:\Users\Administrator\Desktop\python\关键字抓取2021.3.30.csv', 'r', newline = '\n')as g:
#        for row in csv.reader(g):
#            print(row[9])
    关键字抓取网址表 = pandas.read_excel(r'C:\Users\Administrator\Desktop\python\关键字抓取2024.4.10-1.xlsx',sheet_name='Sheet1',header=None) #, nrows=50
    wb = load_workbook('批量爬虫2021.5.24.xlsx')
    sheet = wb.active
    for row in 关键字抓取网址表.values:
#        wb = load_workbook('批量爬虫2021.5.24.xlsx')
#        sheet = wb.active
        #if起到判断开关的作用
#        count = 0
        if row[9] == "p = MixPage()" :
            count +=1
            print(count)
            print(row[9])
            原网站名 = row[6]
            主页链接 = row[7]
            定位xpath = row[10]
#            r = Recorder('批量爬虫2021.5.24.xlsx', 50)  # 50表示每50条记录写入一次文件
            try:
#                '''
                p.get(主页链接) 
                #p.driver.minimize_window()
                网站名 = p.title
                一级网址 = p.url
    #                print(网站名)
    #                print(一级网址)
        #        p.to_iframe('iframe')
    #                x = '//*[@id="lab-show"]/div[2]/div[2]/div/ul/li/a'
                datas = p.eles('xpath:'+定位xpath)
#                r = Recorder('批量爬虫2021.5.24.xlsx', 1)  # 50表示每50条记录写入一次文件
                for data in datas:
                    内容标题 = data.text.replace(' ', '').replace('\n', '').replace('\r', '')
                    内容网址 = data.link
                    sheet.append((now,today,网站名,一级网址,内容标题,内容网址))
    #                print(网站名)
    #                print(一级网址)
    #                print(内容标题)
    #                print(内容网址)
                    
#                    r = Recorder('批量爬虫2021.5.24.xlsx', 1)  # 50表示每50条记录写入一次文件
#                    r.add_data((now,today,网站名,一级网址,内容标题,内容网址))  # 插入一条数据（也可一次插入多条）
    #                print()
    #                    print(内容标题)
    #                    print(内容网址)
        
                    #把新链接写入热榜CSV文件内
        #            with open('C:\\Users\\Administrator\\Desktop\\批量爬虫2021.5.24.csv', 'a+', newline = '\n',encoding='utf-8')as f:
    #                with open(r'C:\Users\Administrator\Desktop\python\批量爬虫2021.5.24.csv', 'a+', newline = '\n')as f:
    #                    w = csv.writer(f)
    #                    w.writerow([now,today,网站名,一级网址,内容标题,内容网址])
            except:
                print('出错跳过了'+row[6])
                with open(r'C:\Users\Administrator\Desktop\python\批量爬虫失败日志2021.6.11.csv', 'a+', newline = '\n')as f:
                        w = csv.writer(f)
                        w.writerow([now,today,原网站名,主页链接])
    wb.save('批量爬虫2021.5.24.xlsx')
    p.close_driver()
#            '''
            
#main无头浏览器()
main谷歌浏览器()

2、快速定位xpath-浏览器F12

在a的位置右键，复制完整的xpath

删除【】中的数字

# -*- coding: utf-8 -*-
"""
Created on Thu Jan 28 10:08:38 2021

@author: Administrator
"""
# from DrissionPage import MixPage
from DrissionPage import WebPage
from DrissionPage import SessionPage
import re
from time import sleep
import csv
import os, sys
import datetime
import pysnooper
from time import sleep
from tenacity import retry, stop_after_attempt, wait_fixed, stop_after_delay
import pandas
from DataRecorder import Recorder  # 记录器
from openpyxl import load_workbook

#now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
now = datetime.datetime.now()
today = datetime.date.today()
yesterday = str(today - datetime.timedelta(days=1))
the_day_before_yesterday = str(today - datetime.timedelta(days=2))
today_4 = str(today - datetime.timedelta(days=3))


# 主页链接 = "http://nncz.nanning.gov.cn/zfxxgkzl/fdzdgknr/ywzt/zfcgjdgl_51300/zczn/"
# 主页链接 = "http://nncz.nanning.gov.cn/zfxxgkzl/fdzdgknr/zcwj/qtzcwj/"
# 主页链接 = "http://zfcg.czj.gxgg.gov.cn/site/category?parentId=3083&childrenCode=ggcgNoticeNews&utm=luban.luban-PC-40673.1069-pc-wsg-ArticlePurchaseNoticeList-front.1.928a3df0f55611eebe0a99024165abd0"
# 主页链接 = "http://zfcg.czj.guilin.gov.cn/site/category?parentId=139012&childrenCode=glCategory112&utm=site.site-PC-40863.1069-pc-wsg-ArticlePurchaseNoticeList-front.1.9c32cd10f55e11ee8796c17ff936f726"
主页链接 = "https://www.mohurd.gov.cn/zhengcefabu/index.html"

# 定位xpath =  '/html/body/div[1]/div/div[4]/div/div[2]/div[2]/div[2]/div/div[1]/ul/li/a'
定位xpath =  '/html/body/div[1]/div[1]/div[2]/div[3]/div/ul/li/a'

#@retry(stop=stop_after_attempt(2),wait=wait_fixed(2))
#@pysnooper.snoop(normalize= True,prefix="主程序")
def main无头浏览器():
#获取主页
    # p = MixPage('s')
    # p = WebPage()
    p = SessionPage()
    count = 0
    try:
        p.get(主页链接) 
        #p.driver.minimize_window()
        网站名 = p.title
        一级网址 = p.url
    #                print(网站名)
    #                print(一级网址)
    #        p.to_iframe('iframe')
    #                x = '//*[@id="lab-show"]/div[2]/div[2]/div/ul/li/a'
        datas = p.eles('xpath:'+定位xpath)
    #                r = Recorder('批量爬虫2021.5.24.xlsx', 1)  # 50表示每50条记录写入一次文件
        for data in datas:
            内容标题 = data.text.replace(' ', '').replace('\n', '').replace('\r', '')
            内容网址 = data.link
            count +=1
            print(count)
            # sheet.append((now,today,网站名,一级网址,内容标题,内容网址))
    #                    r = Recorder('批量爬虫2021.5.24.xlsx', 1)  # 50表示每50条记录写入一次文件
    #                    r.add_data((now,today,网站名,一级网址,内容标题,内容网址))  # 插入一条数据（也可一次插入多条）
            print(内容标题)
            print(内容网址)
    
            #把新链接写入热榜CSV文件内
    #            with open('C:\\Users\\Administrator\\Desktop\\批量爬虫2021.5.24.csv', 'a+', newline = '\n',encoding='utf-8')as f:
    #                    with open(r'C:\Users\Administrator\Desktop\python\批量爬虫2021.5.24.csv', 'a+', newline = '\n')as f:
    #                        w = csv.writer(f)
    #                        w.writerow([now,today,网站名,一级网址,内容标题,内容网址])
    except:
        print('出错跳过了')
    #         with open(r'C:\Users\Administrator\Desktop\python\批量爬虫失败日志2021.6.11.csv', 'a+', newline = '\n')as f:
    #                 w = csv.writer(f)
    #                 w.writerow([now,today,原网站名,主页链接])
    # wb.save('批量爬虫2021.5.24.xlsx')
    # p.close_driver()
    #            '''
main无头浏览器()




print('-'*200)
#@retry(stop=stop_after_attempt(2),wait=wait_fixed(2))
def main谷歌浏览器():
#获取主页
    # p = MixPage()
    p = WebPage()

    count = 0
    try:
#                '''
        p.get(主页链接) 
        #p.driver.minimize_window()
        网站名 = p.title
        一级网址 = p.url
#                print(网站名)
#                print(一级网址)
#        p.to_iframe('iframe')
#                x = '//*[@id="lab-show"]/div[2]/div[2]/div/ul/li/a'
        datas = p.eles('xpath:'+定位xpath)
#                r = Recorder('批量爬虫2021.5.24.xlsx', 1)  # 50表示每50条记录写入一次文件
        for data in datas:
            内容标题 = data.text.replace(' ', '').replace('\n', '').replace('\r', '')
            内容网址 = data.link
            count +=1
            print(count)
            # sheet.append((now,today,网站名,一级网址,内容标题,内容网址))
#                print(网站名)
#                print(一级网址)
            
            print(内容标题)
            print(内容网址)
            
#                    r = Recorder('批量爬虫2021.5.24.xlsx', 1)  # 50表示每50条记录写入一次文件
#                    r.add_data((now,today,网站名,一级网址,内容标题,内容网址))  # 插入一条数据（也可一次插入多条）
#                print()
#                    print(内容标题)
#                    print(内容网址)

            #把新链接写入热榜CSV文件内
#            with open('C:\\Users\\Administrator\\Desktop\\批量爬虫2021.5.24.csv', 'a+', newline = '\n',encoding='utf-8')as f:
#                with open(r'C:\Users\Administrator\Desktop\python\批量爬虫2021.5.24.csv', 'a+', newline = '\n')as f:
#                    w = csv.writer(f)
#                    w.writerow([now,today,网站名,一级网址,内容标题,内容网址])
    except:
        print('出错跳过了')
#             with open(r'C:\Users\Administrator\Desktop\python\批量爬虫失败日志2021.6.11.csv', 'a+', newline = '\n')as f:
#                     w = csv.writer(f)
#                     w.writerow([now,today,原网站名,主页链接])
# wb.save('批量爬虫2021.5.24.xlsx')
# p.close_driver()
#            '''
            
# main无头浏览器()
main谷歌浏览器()

本文参与腾讯云自媒体同步曝光计划，分享自微信公众号。

原始发表：2024-04-10，如有侵权请联系 cloudcommunity@tencent.com 删除

网站