问IndexError:超出范围的列表索引(在Reddit数据爬虫上)

EN

def get_submission_ids(self, pages=2):
    return [
        submission.id
        for submission in self.reddit.subreddit(self.subreddit).hot(
            limit=25 * pages
        )
    ]

def get_submissions(self, pages=2):
    return list(self.reddit.subreddit(self.subreddit).hot(limit=25 * pages))

def save_comments_submissions(self, pages):
    """
        1. Save all the ids of submissions.
        2. For each submission, save information of this submission. (submission_id, #comments, score, subreddit, date, title, body_text)
        3. Save comments in this submission. (comment_id, submission_id, author, date, score, body_text)
        4. Separately, save them to two csv file.
        Note: You can link them with submission_id.
        Warning: According to the rule of Reddit API, the get action should not be too frequent. Safely, use the defalut time span in this crawler.
    """

    print("Start to collect all submission ids...")
    submissions = self.get_submissions(pages)
    print(
        "Start to collect comments...This may cost a long time depending on # of pages."
    )
    comments = []
    pandas_submissions = []
    for count, submission in enumerate(submissions):
        pandas_submissions.append(
            (
                submission.name[3:],
                submission.num_comments,
                submission.score,
                submission.subreddit_name_prefixed,
                date.fromtimestamp(submission.created_utc).isoformat(),
                submission.title,
                submission.selftext,
            )
        )
        temp_comments = self.get_comments(submission)
        comments += temp_comments
        print(str(count) + " submissions have got...")

    comments_fieldnames = [
        "comment_id",
        "submission_id",
        "author_name",
        "post_time",
        "comment_score",
        "text",
    ]
    df_comments = pd.DataFrame(comments, columns=comments_fieldnames)
    df_comments.to_csv("comments.csv")
    submissions_fieldnames = [
        "submission_id",
        "num_of_comments",
        "submission_score",
        "submission_subreddit",
        "post_date",
        "submission_title",
        "text",
    ]
    df_submission = pd.DataFrame(pandas_submissions, columns=submissions_fieldnames)
    df_submission.to_csv("submissions.csv")

from datetime import date
import sys


import pandas as pd
import praw


class Crawler:
    """
        basic_url is the reddit site.
        headers is for requests.get method
        REX is to find submission ids.
    """

    def __init__(self, subreddit="apple"):
        """
            Initialize a Crawler object.
                subreddit is the topic you want to parse. default is r"apple"
            basic_url is the reddit site.
            headers is for requests.get method
            REX is to find submission ids.
            submission_ids save all the ids of submission you will parse.
            reddit is an object created using praw API. Please check it before you use.
        """
        self.subreddit = subreddit
        self.submission_ids = []
        self.reddit = praw.Reddit(
            client_id="your_id",
            client_secret="your_secret",
            user_agent="subreddit_comments_crawler",
        )

    def get_submissions(self, pages=2):
        """
            Collect all submissions..
            One page has 25 submissions.
            page url: https://www.reddit.com/r/subreddit/?count25&after=t3_id
                id(after) is the last submission from last page.
        """
        return list(self.reddit.subreddit(self.subreddit).hot(limit=25 * pages))

    def get_comments(self, submission):
        """
            Submission is an object created using praw API.
        """
        #         Remove all "more comments".
        submission.comments.replace_more(limit=None)
        comments = []
        for each in submission.comments.list():
            try:
                comments.append(
                    (
                        each.id,
                        each.link_id[3:],
                        each.author.name,
                        date.fromtimestamp(each.created_utc).isoformat(),
                        each.score,
                        each.body,
                    )
                )
            except AttributeError as e:  # Some comments are deleted, we cannot access them.
                #                 print(each.link_id, e)
                continue
        return comments

    def save_comments_submissions(self, pages):
        """
            1. Save all the ids of submissions.
            2. For each submission, save information of this submission. (submission_id, #comments, score, subreddit, date, title, body_text)
            3. Save comments in this submission. (comment_id, submission_id, author, date, score, body_text)
            4. Separately, save them to two csv file.
            Note: You can link them with submission_id.
            Warning: According to the rule of Reddit API, the get action should not be too frequent. Safely, use the defalut time span in this crawler.
        """

        print("Start to collect all submission ids...")
        submissions = self.get_submissions(pages)
        print(
            "Start to collect comments...This may cost a long time depending on # of pages."
        )
        comments = []
        pandas_submissions = []
        for count, submission in enumerate(submissions):
            pandas_submissions.append(
                (
                    submission.name[3:],
                    submission.num_comments,
                    submission.score,
                    submission.subreddit_name_prefixed,
                    date.fromtimestamp(submission.created_utc).isoformat(),
                    submission.title,
                    submission.selftext,
                )
            )
            temp_comments = self.get_comments(submission)
            comments += temp_comments
            print(str(count) + " submissions have got...")

        comments_fieldnames = [
            "comment_id",
            "submission_id",
            "author_name",
            "post_time",
            "comment_score",
            "text",
        ]
        df_comments = pd.DataFrame(comments, columns=comments_fieldnames)
        df_comments.to_csv("comments.csv")
        submissions_fieldnames = [
            "submission_id",
            "num_of_comments",
            "submission_score",
            "submission_subreddit",
            "post_date",
            "submission_title",
            "text",
        ]
        df_submission = pd.DataFrame(pandas_submissions, columns=submissions_fieldnames)
        df_submission.to_csv("submissions.csv")


if __name__ == "__main__":
    args = sys.argv[1:]
    if len(args) != 2:
        print("Wrong number of args...")
        exit()

    subreddit, pages = args
    c = Crawler(subreddit)
    c.save_comments_submissions(int(pages))

>>> ids = []
>>> ids[-1]
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
IndexError: list index out of range
>>> ids = ['1']
>>> ids[-1]
'1'