Python用Pyspider爬取TripAdvisor的景点信息

先上效果图

上代码：

from pyspider.libs.base_handler import *
import pymongo

class Handler(BaseHandler):
    crawl_config = {
    }

    client = pymongo.MongoClient('localhost')
    db = client['TripAdvise']

    @every(minutes=24 * 60)
    def on_start(self):
        self.crawl('https://www.tripadvisor.cn/Attractions-g186338-Activities-c47-London_England.html',callback=self.index_page, validate_cert=False)

    @config(age=10 * 24 * 60 * 60)
    def index_page(self, response):
        for each in response.doc('div.listing_title > a').items():
            self.crawl(each.attr.href, callback=self.detail_page, validate_cert=False)
        # 翻页
        next = response.doc('.pagination .nav.next').attr.href
        self.crawl(next, callback=self.index_page, validate_cert=False)

    @config(priority=2)
    def detail_page(self, response):
        url = response.url
        name = response.doc('.h1').text()
        rating = response.doc(' div.ratingContainer > a > span').text()
        garde = response.doc('div.section.rating > span').text()
        address = response.doc('.contactInfo > .address').text()
        phone = response.doc('div.contact > div.contactType.phone.is-hidden-mobile > div').text()
        opening = response.doc('div.prw_rup.prw_common_atf_header_bl_responsive.headerBL > div > span').text()
        introduction = response.doc('.centerWell > div > div > div > div > div > span').text()
        return {
            "url": url,
            "name": name,
            "rating": rating,
            'garde': garde,
            "address": address,
            "phone": phone,
            "opening": opening,
            "introduction": introduction
        }

    def on_result(self, result):
        if result:
            self.save_to_mongo(result)

    def save_to_mongo(self, result):
        if self.db['london'].insert(result):
            print('保存到MongoDB成功', result)

参考：https://cuiqingcai.com/8317.html