python学习(26) 分析Ajax爬取今日头条cosplay美女

分析ajax请求格式,模拟发送http请求,从而获取网页代码,进而分析取出需要的数据和图片。这里分析ajax请求,获取cosplay美女图片。

登陆今日头条,点击搜索,输入cosplay

https://cdn.llfc.club/py261.png
下面查看浏览器F12,点击XHR,这里能截取ajax请求,由于已经请求过该页面,所以点击F5,刷新,如下图
https://cdn.llfc.club/py262.png
下面我们点击name下的链接,查看headers看到请求信息
https://cdn.llfc.club/py263.png
可以看到请求的url为
https://www.toutiao.com/search_content/?offset=0&format=json&keyword=cosplay&autoload=true&count=20&cur_tab=1&from=search_tab&pd=synthesis
offset为0,表示当前页面的偏移量,我试着向下滑动页面,name下加载出很多连接,offset每次递增20,keyword为cosplay,是我们搜索的关键词,count表示图集的数量,其他的都不变。
所以我们可以构造一个http请求,包含上面的格式。接下来看看preview的内容
https://cdn.llfc.club/py264.png
data就是页面加载出来的图片文章列表
点击其中一个data,查看
https://cdn.llfc.club/py265.png
可以看得出图片列表和large图片相差的就是list和large的区别,如
http://p1-tt.bytecdn.cn/list/97e000601ee89d997af"为缩略图
http://p1-tt.bytecdn.cn/large/97e000601ee89d997af"为大图
所以只需将list替换为larg即可。之后发送http请求,获取对应的图片即可。下面为完整代码

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
import requests
import re
import time
from pyquery import PyQuery as pq
from urllib.parse import urlencode
import os
USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.221 Safari/537.36 SE 2.X MetaSr 1.0'
COOKIES = r'UM_distinctid=167005cc741184-02d14e2fd49b05-10724c6f-1fa400-167005cc74263a; uuid="w:5a4841d5dda248c389f83e5b9c57608a"; sso_uid_tt=45b83f6c549dabd4248c611aa98222d3; toutiao_sso_user=4853db5c812f7bcb367cdcbef0967d06; sso_login_status=1; login_flag=54c7da4daac99058f7b6e4b8975cca01; sessionid=5e86f286970804ed8fd11abf0bf328e4; uid_tt=dd5e177a1cbf9746eb634307d64afd22; sid_tt=5e86f286970804ed8fd11abf0bf328e4; sid_guard="5e86f286970804ed8fd11abf0bf328e4|1541899310|15552000|Fri\054 10-May-2019 01:21:50 GMT"; tt_webid=6622406673465067021; WEATHER_CITY=%E5%8C%97%E4%BA%AC; cp=5BE7386A29798E1; tt_webid=75478811657; __tasessionId=qwly22e8r1541906736076; CNZZDATA1259612802=1520919144-1541896080-https%253A%252F%252Fwww.baidu.com%252F%7C1541901480; csrftoken=b970f054ea259eb162e572217e6756ca'
REFER = 'https://www.toutiao.com/search/?keyword=cosplay'
class AjaxScrapy(object):
def __init__(self,pages=1):
try:
self.m_session = requests.Session()
self.m_headers = {'User-Agent':USER_AGENT,
'referer':REFER,
}

self.m_cookiejar = requests.cookies.RequestsCookieJar()
self.dirpath = os.path.split(os.path.abspath(__file__))[0]
for cookie in COOKIES.split(';'):
key,value = cookie.split('=',1)
self.m_cookiejar.set(key,value)
except:
print('init error!!!')
def getOffset(self,index=0):
try:
params = {
'offset':str(20*index),
'format':'json',
'keyword':'cosplay',
'autoload':'true',
'count':'20',
'cur_tab':'3',
'from':'gallery',
}
httpaddr = 'https://www.toutiao.com/search_content/?'+urlencode(params)
req = self.m_session.get(httpaddr,headers=self.m_headers, cookies=self.m_cookiejar, timeout=5)
if req.status_code != 200:
return None
res = req.json()
if('data' not in res.keys()):
return None
for item in res.get('data'):
if('title' not in item.keys()):
continue
if('image_list' not in item.keys()):
continue
titlenamelist=item['title'].split('/')
titlename = titlenamelist[-1]
savedir=os.path.join(self.dirpath,titlename)
print('正在抓取'+titlename+'.........')
if(os.path.exists(savedir) == False):
os.makedirs(savedir)
imagelist = item.get('image_list')
for imag in imagelist:
if 'url' not in imag.keys():
continue
#print(imag['url'])
image1,image2=imag['url'].split('list')
image3=imag['url'].split('/')[-1]
imagepath=os.path.join(savedir,image3+'.jpg')
if(os.path.exists(imagepath)):
continue
imageaddr = 'http:'+image1+'large'+image2
imagedata=self.m_session.get(imageaddr,timeout=5)
with open (imagepath,'wb')as f:
f.write(imagedata.content)
print('抓取'+titlename+'成功!!!.........')
time.sleep(1)
return req.json()
except:
print('get over view error')
return None

if __name__ == "__main__":
try:
asscrapy = AjaxScrapy()
for i in range(0,5):
res = asscrapy.getOffset(i)
if(res == None):
continue
#print(type(res))

except:
print('scrapy exception!')
pass

更多源码下载;
https://github.com/secondtonone1/python-/tree/master/pythoncookie
个人博客
https://www.limerence2017.com
谢谢关注我的公总号:
https://cdn.llfc.club/gzh.jpg