爬取虎扑网步行街帖子信息并写入CSV文件

#!/usr/bin/env python3
# -*- coding:utf-8 -*-

# &&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&
#   @Version: v1.0
#   @License: Apache Licence 2.0
#   @File Name: hupubxj.py
#   @Description: 爬取虎扑步行街的帖子信息并写入csv文件
#   @Author: pengshp <[email protected]>
#   @Date: 2018/10/19 0019
# &&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&

import requests
from bs4 import BeautifulSoup
import bs4
import csv


def getHTML(link):
    """获取网站HTML"""
    kv = {'User-Agent': 'Mozilla/5.0'}
    try:
        r = requests.get(link, headers=kv)
        r.raise_for_status()
        html = r.text
    except requests.HTTPError:
        print("爬取异常")
    return html


def getData(html):
    """获取帖子的详细信息"""
    data_list = []
    soup = BeautifulSoup(html, 'lxml')
    ul = soup.find_all('ul', class_="for-list")
    for li in ul[0].children:
        if isinstance(li, bs4.element.Tag):
            title_box = li.select('a[class="truetit"]')
            title = title_box[0].string
            url = 'https://bbs.hupu.com%s' % title_box[0]['href']
            author = li.select('a[class="aulink"]')[0].string
            time = li.select('a[style="color:#808080;cursor: initial; "]')[0].string
            relay_view = li.select('span[class="ansour box"]')[0].string
            relay = relay_view.split('/')[0].strip()
            view = relay_view.split('/')[1].strip()
            endreply = li.select('div[class="endreply box"]')[0].a.string
            endauthor = li.select('div[class="endreply box"]')[0].span.string
            data_list.append([title, url, author, time,
                              relay, view, endreply, endauthor])
    return data_list


if __name__ == '__main__':
    link = 'https://bbs.hupu.com/bxj'
    page = getHTML(link)
    datas = getData(page)
    for data in datas:
        with open('bxj.csv', 'a+', encoding='utf-8', newline='') as f:
            w = csv.writer(f)
            w.writerow(data)
    print("数据写入完成！")