使用python实现简单的爬虫操作

requests模块

安装
pip install requests

抓包工具使用

比如Chrome浏览器
按F12-》Network-》点击XHR-》刷新页面-》选择下方数据包
就可以看到相关数据

1.指定url

UA伪装

headers 请求头
User-Agent 用户代理，一般是浏览器
示例：
这是从我的chrome浏览器的抓包工具中查看的，User-Agent字段

headers = {
    'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.75 Safari/537.36'
}

请求参数
params 对于get操作
data 对于post

2.发起请求

requests.get(url,params,headers)
requests.post(url,data,headers)
一般直接请求是get多，ajax请求post多

3.获取响应数据

response = requests.get(url,params,headers)
response.text 文本数据
response.json() ajax数据
查看是什么数据Content-Type
application/json ---》json数据
application/x-www-form-urlencoded ---》text数据

4.持久化存储

json.dump() 存储json对象
f.write() 存储text数据或者json字符串
后续用数据库存储

案例

案例1：爬取搜狗首页的页面数据

import requests
if __name__ == "__main__":
    # 1. 设定url
    url = "https://www.sogou.com/"
    # 2.发起请求
    response = requests.get(url=url)
    # 3.获取响应数据
    page_text = response.text
    # 4.持久化存储
    with open("./sougou.html","w",encoding="utf-8") as f:
        f.write(page_text)

案例2：爬取搜狗关键字搜索

#使用ua伪装请求头headers  使用params 参数
import requests
if __name__ == "__main__":
    # 1.拼接url
    url = 'https://www.sogou.com/web'
    # 1.1 做一个参数列表
    kw = input(':>>')
    params = {
        'query':kw
    }
    # 2.做请求头
    headers = {
        'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.75 Safari/537.36'
    }
    # 发送请求
    response = requests.get(url=url,params=params,headers=headers)
    # 解析数据
    page_text = response.text
    # 持久化存储
    fileName = kw + '.html'
    with open(fileName,"w",encoding="utf-8") as f:
        f.write(page_text)
    print(fileName,"下载成功")

案例3：爬取百度翻译

import requests
import json
if __name__ == "__main__":
    post_url = "https://fanyi.baidu.com/sug"
    headers = {
        "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36"
    }
    msg = input(">>:")
    data = {
    "kw":msg
    }
    response = requests.post(url=post_url,data=data,headers=headers)
    data_json = response.json()
    fileName = msg + ".json"
    fp = open(fileName,"w",encoding="utf-8")
    json.dump(data_json,fp=fp,ensure_ascii=False)
    print("写入成功")

案例4：爬取豆瓣电影

import requests
import json
if __name__ == "__main__":
    url = "https://movie.douban.com/j/chart/top_list"
    params = {
        'type':'11',
        'interval_id': '100:90',
        'action': '',
        'start': '0',
        'limit': '20',
    }
    headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36'
    }
    response = requests.get(url=url,params=params,headers=headers)
    #解析字符串
    movie_list = response.json()
    #持久化存储
    fp = open("./movie.json","w",encoding="utf-8")
    json.dump(movie_list,fp=fp,ensure_ascii=False)
    print("爬取成功")

案例5：爬取所有北京的肯德基餐厅信息

import requests
import os
if __name__ == "__main__":
    post_url = "http://www.kfc.com.cn/kfccda/ashx/GetStoreList.ashx?op=keyword"
    headers = {
        "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36"
    }
    data = {
        "cname":"",
        "pid": "",
        "keyword": "北京",
        "pageIndex": "1",
        "pageSize": "10"
    }
    start = 1
    end = 8
    if not os.path.exists('./beijing'):
        os.mkdir('./beijing')
    for i in range(start,end+1):
        data['pageIndex'] = str(i)
        # 发送post请求
        response = requests.post(url=post_url,data=data,headers=headers)
        response_json_str = response.text
        fileName = "./beijing/beijing%s.json" % i
        fp = open(fileName,"w",encoding="utf-8")
        fp.write(response_json_str)
        print("爬取成功%s" % i)

案例六：爬取药监局详情页数据

#爬取药监局的信息
import requests
import json
import os
import time
import random
if __name__ == "__main__":
    # 发送post请求，获取首页ajax信息
    index_url = 'http://125.35.6.84:81/xk/itownet/portalAction.do?method=getXkzsList'
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36"
    }
    #修改page，就可以获取每页数据
    index_data = {
        "on": "true",
        "page": "1",
        "pageSize": "15",
        "productName":"",
        "conditionType":"1",
        "applyname":"",
        "applysn":""
    }
    #将每页数据存储到一个列表中
    max_page_num = 20
    for page_num in range(1,max_page_num):  #爬取每一页数据
        index_data['page'] = page_num
        index_json = requests.post(url=index_url,data=index_data,headers=headers).json()
        #解析每一页的企业列表
        enterprise_id_list = []
        enterprise_list = index_json['list']
        # print(enterprise_list)

        #爬取详情页数据
        desc_url = 'http://125.35.6.84:81/xk/itownet/portalAction.do?method=getXkzsById'
        #需要拼接data
        desc_data = {
            "id":""
        }
        pre_page_data = []
        for enterprise in enterprise_list:
            desc_data["id"] = enterprise["ID"]
            # print(desc_data)
            desc_json = requests.post(url=desc_url,data=desc_data,headers=headers).json()
            # 将获取的json字符串放到一起
            pre_page_data.append(desc_json)
        #持久化存储
        if not os.path.exists('./药监局数据'):
            os.mkdir('./药监局数据')
        fileName = "./药监局数据/page%d.json"%page_num
        with open(fileName,"w",encoding="utf-8") as fp:
            json.dump(pre_page_data,fp=fp,ensure_ascii=False)
        print("page%d爬取完成"%page_num)
        d = random.random()
        print("下一次爬取时间",d)
        time.sleep(d)

d = random.random()
print("下一次爬取时间",d)
time.sleep(d)

可以没有，这个是模拟人的行为