问无法使用scrapy从第二页中抓取数据

EN

504 Gateway Time-out

class Shopee(scrapy.Spider):
  name = 'shopee'

  script = '''
    function main(splash, args)
      assert(splash:go(args.url))
      assert(splash:wait(5.0))
      treat=require('treat')
      result = {}
      pages = splash:select('.shopee-mini-page-controller__total')

      for i=1,3,1 do
        for j=1,2,1 do
          assert(splash:runjs("window.scrollBy(0, 1300)"))
          assert(splash:wait(5.0))
        end

        result[i]=splash:html()
        assert(splash:runjs('document.querySelector(".shopee-icon-button--right").click()'))
        assert(splash:wait(8.0))
      end
      return treat.as_array(result)
    end
  '''

  def start_requests(self):
    urls = [
        'https://shopee.sg/search?keyword=hdmi'
    ]
    for link in urls:
      yield SplashRequest(url=link, callback=self.parse, endpoint='execute', args={'wait': 2.5, 'lua_source' : self.script}, dont_filter=True)


  def parse(self, response):
    for page in response.data:
      sel = Selector(text=page)
      yield {
        'urls': sel.xpath("//div[contains(@class, 'shopee-search-item-result__item')]//a[*]/@href").getall()
      }