问如何在Python2.7中创建一个有效的爬虫

EN

import urllib
import urlparse
from itertools import ifilterfalse
from urllib2 import URLError, HTTPError

from bs4 import BeautifulSoup

urls = {"http://www.kiabi.es/"}
visited = set()


def get_html_text(url):
    try:
        return urllib.urlopen(current_url.encode('ascii','ignore')).read()
    except (IOError,URLError, HTTPError, urllib.ContentTooShortError):
        print "Error getting " + current_url
        urls.add(current_url)


def find_internal_links_in_html_text(html_text, base_url):
    soup = BeautifulSoup(html_text, "html.parser")
    links = set()
    for tag in soup.findAll('a', href=True):
        url = urlparse.urljoin(base_url, tag['href'])
        domain = urlparse.urlparse(base_url).hostname
        if domain in url:
            links.add(url)
    return links


def is_url_already_visited(url):
    return url in visited


while urls:
  try:
    word = '#C'
    current_url = urls.pop()
    print "Parsing", current_url
    if word in current_url:


        print "Parsing", current_url
        htmltext= urllib.urlopen(current_url).read()
        soup= BeautifulSoup(htmltext)

        [get the product info and save it into a sql database]

    html_text = get_html_text(current_url)
    visited.add(current_url)
    found_urls = find_internal_links_in_html_text(html_text, current_url)
    new_urls = ifilterfalse(is_url_already_visited, found_urls)
    urls.update(new_urls)

except Exception:
    pass