更新携程网站分析
最新推荐文章于 2024-08-08 15:47:54 发布
咿咿呀呀py
最新推荐文章于 2024-08-08 15:47:54 发布
阅读量2k
收藏
11
点赞数
4
最新推荐文章于 2024-08-08 15:47:54 发布
阅读量2k
收藏
11
点赞数
4
博主几天前发布的文章，几天后又有了新的解决思路：
完整的代码放在文章最后 用手机端抓取
首先请求到这个页面
我们的目标抓取景点名 详情链接 评分 景区等级 门票价格等数据
如果我们选择在主页面（上面这个页面）抓取 景点名和详情链接都可以捕获到，但是有的景区没有评分和等级-----------所以造成的困惑是后期用zip方法处理数据没有办法一一对应，因此我抓到详情链接 然后再通过每一个的详情链接去请求抓取需要的数据
1.先请求主页面得到详情链接
2.通过详情链接去请求子页面
兄弟们大坑来了---------得到的详情链接请求的子页面数据是动态加载的，也就意味着我们没有办法抓到评分等级这类的数据
我是这样做的：
先随便点开一个子链接 按F12 打开抓包工具
这个包的请求参数 poild 要改变
这个poild可以在详情页链接得到 于是我用正则将他匹配下来，带着他去请求这个url
这个数据包里只有景区评分和景区等级 没有景区价格 但是景区价格也是同样的方法，先找到这个包，然后带着poild去匹配就可以了
现在需求升级：
我现在需要全国的景区景点 很简单，先检索到全国的 城市id 然后改变参数
只要改变这个参数就可以
城市id我选择从景区景点介入
有大坑来了
这个数据包所返回的是字符串，没有办法用
返回的数据因为pycharm的原因是个字符串
解决方法：
用正则匹配 代码
momos = re.findall(r'"districtId":(.*?),.*?"districtName":"(.*?)",',data,re.S) |
城市id也有了，看我最后的代码
import mysql.connector
import requests
import json
from jsonpath import jsonpath
import sys
import re
import random
import time
from fake_useragent import UserAgent
class XCspider():
def __init__(self):
self.randomtime = random.randint(1,2)
self.UA = UserAgent()
self.conn = mysql.connector.connect(host='localhost',user='root',password='123456',
database='sjb',auth_plugin='mysql_native_password')
self.mysql_cursor = self.conn.cursor()
ip = [
# '115.209.115.169:4256',
# '117.93.79.97:4232',
# '117.87.64.192:4231',
# '114.99.13.191:4226',
# '36.6.69.198:4278',
# '59.58.49.214:4231',
# '121.61.195.9:4231',
# '115.229.206.57:4231',
# '175.162.210.135:4231',
# '120.35.176.91:4213',
# '110.230.217.127:4223',
# '60.173.24.129:4225',
# '183.92.219.224:4225',
# '223.156.86.132:4231',
# '1.199.199.253:4247',
# '42.59.111.242:4285',
# '121.226.152.98:4236',
# '27.157.230.186:4216',
# '222.77.213.163:4245',
# '220.168.239.10:4231',
# '113.218.241.238:4231',
# '1.49.231.217:4267',
# '117.95.200.103:4231',
# '27.44.37.158:4231',
# '122.241.27.32:4231',
'113.235.166.147:4278',
'112.85.232.108:4278',
'49.88.149.57:4263',
'106.110.198.7:4236',
]
self.ip = {
'https':random.choice(ip)
}
self.url = "https://m.ctrip.com/restapi/soa2/18254/json/getAttractionList"
self.payload = {
"index": 1,
"count": 20,
"sortType": 1,
"isShowAggregation": True,
"districtId": 2,
"scene": "DISTRICT",
"pageId": "214062",
"traceId": "1dcf2c53-e847-7d59-c4a9-637959404701",
"extension": [
{
"name": "osVersion",
"value": "6.0"
},
{
"name": "deviceType",
"value": "android"
}
],
"filter": {
"filterItems": []
},
"crnVersion": "2020-09-01 22:00:45",
"isInitialState": True,
"head": {
"cid": "09031137218199825303",
"ctok": "",
"cver": "1.0",
"lang": "01",
"sid": "8888",
"syscode": "09",
"xsid": "",
"extension": []
}
}
self.headers = {
'Connection': 'keep-alive',
'cookieOrigin': 'https://m.ctrip.com',
# 'User-Agent': random.choice(self.UA),
'User-Agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Mobile Safari/537.36',
'content-type': 'application/json',
'Accept': '*/*',
'Origin': 'https://m.ctrip.com',
'Sec-Fetch-Site': 'same-origin',
'Sec-Fetch-Mode': 'cors',
'Sec-Fetch-Dest': 'empty',
'Referer': 'https://m.ctrip.com/webapp/you/gspoi/sight/2.html?seo=0&allianceid=4897&sid=130026&isHideNavBar=YES&from=https%3A%2F%2Fm.ctrip.com%2Fwebapp%2Fyou%2Fgsdestination%2Fplace%2F2.html%3Fseo%3D0%26ishideheader%3Dtrue%26secondwakeup%3Dtrue%26dpclickjump%3Dtrue%26allianceid%3D4897%26sid%3D130026%26from%3Dhttps%253A%252F%252Fm.ctrip.com%252Fhtml5%252F',
'Accept-Language': 'zh-CN,zh;q=0.9',
'Cookie': 'GUID=09031137218199825303; MKT_CKID=1655441050631.y3cpg.1v7v; _RSG=.mK7Vo8KIkFkJFbXO64ek8; _RGUID=fb92508c-9f1d-4c3e-8ba8-10662194f4ee; _RDG=2894d2e5596bd8261b1bd4f0d138733ac9; _bfaStatusPVSend=1; nfes_isSupportWebP=1; nfes_isSupportWebP=1; _ga=GA1.2.1595398136.1655441061; ibu_h5_lang=en; ibu_h5_local=en-us; StartCity_Pkg=PkgStartCity=25; ibulanguage=CN; ibulocale=zh_cn; cookiePricesDisplayed=CNY; _gid=GA1.2.2020737530.1655798638; MKT_OrderClick=ASID=4897130026&AID=4897&CSID=130026&OUID=&CT=1655883035853&CURL=https%3A%2F%2Fm.ctrip.com%2Fwebapp%2Fyou%2Fgspoi%2Fsight%2F2.html%3Fseo%3D0%26allianceid%3D4897%26sid%3D130026%26isHideNavBar%3DYES%26from%3Dhttps%253A%252F%252Fm.ctrip.com%252Fwebapp%252Fyou%252Fgsdestination%252Fplace%252F2.html%253Fseo%253D0%2526ishideheader%253Dtrue%2526secondwakeup%253Dtrue%2526dpclickjump%253Dtrue%2526allianceid%253D4897%2526sid%253D130026%2526from%253Dhttps%25253A%25252F%25252Fm.ctrip.com%25252Fhtml5%25252F&VAL={"h5_vid":"1655441049907.gadsg"}; Session=smartlinkcode=U130026&smartlinklanguage=zh&SmartLinkKeyWord=&SmartLinkQuary=&SmartLinkHost=; MKT_CKID_LMT=1655959369643; __zpspc=9.8.1655959369.1655959369.1%232%7Cwww.baidu.com%7C%7C%7C%7C%23; _jzqco=%7C%7C%7C%7C1655959370213%7C1.828646019.1655441050624.1655606559937.1655959369657.1655606559937.1655959369657.undefined.0.0.10.10; _bfs=1.1; _RF1=183.253.26.192; appFloatCnt=1; _bfi=p1%3D102001%26p2%3D0%26v1%3D237%26v2%3D0; _bfaStatus=success; _gat=1; MKT_Pagesource=H5; mktDpLinkSource=ullink; _pd=%7B%22r%22%3A20%2C%22d%22%3A253%2C%22_d%22%3A233%2C%22p%22%3A253%2C%22_p%22%3A0%2C%22o%22%3A254%2C%22_o%22%3A1%2C%22s%22%3A255%2C%22_s%22%3A1%7D; _bfa=1.1655441049907.gadsg.1.1655886903531.1655959404092.24.240.214059; _ubtstatus=%7B%22vid%22%3A%221655441049907.gadsg%22%2C%22sid%22%3A24%2C%22pvid%22%3A240%2C%22pid%22%3A%22214059%22%7D; Union=OUID=&AllianceID=4897&SID=130026&SourceID=&createtime=1655959405&Expires=1656564204559; GUID=09031168218403899804'
}
# 获取所有城市id 还是通过酒店去抓 获取城市id
# def cityId(self,a):
# # 发送请求
# for i in a:
# if i == 21880:
# continue
#
# url = "https://m.ctrip.com/webapp/hotel/j/hoteldetail/dianping/api/static/city"
# headers = {
# 'Cookie': 'JSESSIONID=C49F4E4E2865153D0814E187AE12B13A; GUID=09031168218403899804'
# }
# params = {
# 'oversea': 'false',
# # 更换城市id
# 'index': i,
# 'pageid': 212092
# }
#
# # response = requests.get(url,headers=headers,params=params)
# response = requests.get(url,headers=headers,params=params,proxies=self.ip)
# print('url为',i)
# # print(response.json())
# # 解析数据
# # 得到所有的城市id
# cityId = jsonpath(response.json(),'$..cityId')
# print(cityId)
# # 遍历取值
# for it in cityId:
# print(it)
# self.payload['index'] = 1
# self.payload['districtId'] = it
# # 调用方法
# self.sendUrl()
# 从景点找到的城市接口
def cityID(self):
url = "https://m.ctrip.com/restapi/soa2/17916/json/invokeOnDemand?_fxpcqlniredt=09031137218199825303&x-traceID=09031137218199825303-1656049564481-6585044"
payload = {
"data": "{\"source\":\"destH5\"}",
"serviceName": "DestinationCoreService.getStaticDistrictCategoryList",
"head": {
"cid": "09031137218199825303",
"ctok": "",
"cver": "1.0",
"lang": "01",
"sid": "8888",
"syscode": "09",
"xsid": "",
"extension": []
}
}
headers = {
'cookieOrigin': 'https://m.ctrip.com',
'User-Agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Mobile Safari/537.36',
'content-type': 'application/json',
'Origin': 'https://m.ctrip.com',
'Referer': 'https://m.ctrip.com/webapp/you/gsdestination/citySelector/citySelector.html?seo=0&districtId=2&allianceid=4897&sid=155952&isHideNavBar=YES&from=https%3A%2F%2Fm.ctrip.com%2Fwebapp%2Fyou%2Fgsdestination%2Fplace%2F2.html%3Fseo%3D0%26ishideheader%3Dtrue%26secondwakeup%3Dtrue%26dpclickjump%3Dtrue%26allianceid%3D4897%26sid%3D155952%26ouid%3Dindex%26from%3Dhttps%253A%252F%252Fm.ctrip.com%252Fhtml5%252F',
'Accept-Language': 'zh-CN,zh;q=0.9',
'Cookie': 'GUID=09031137218199825303; MKT_CKID=1655441050631.y3cpg.1v7v; _RSG=.mK7Vo8KIkFkJFbXO64ek8; _RGUID=fb92508c-9f1d-4c3e-8ba8-10662194f4ee; _RDG=2894d2e5596bd8261b1bd4f0d138733ac9; _bfaStatusPVSend=1; nfes_isSupportWebP=1; nfes_isSupportWebP=1; _ga=GA1.2.1595398136.1655441061; ibu_h5_lang=en; ibu_h5_local=en-us; StartCity_Pkg=PkgStartCity=25; ibulanguage=CN; ibulocale=zh_cn; cookiePricesDisplayed=CNY; _gid=GA1.2.2020737530.1655798638; Session=smartlinkcode=U130026&smartlinklanguage=zh&SmartLinkKeyWord=&SmartLinkQuary=&SmartLinkHost=; MKT_Pagesource=H5; _jzqco=%7C%7C%7C%7C1655959370213%7C1.828646019.1655441050624.1655967880712.1655981364160.1655967880712.1655981364160.undefined.0.0.12.12; __zpspc=9.10.1655981364.1655981364.1%232%7Cwww.baidu.com%7C%7C%7C%25E6%2590%25BA%25E7%25A8%258B%7C%23; appFloatCnt=3; _bfi=p1%3D102001%26p2%3D0%26v1%3D272%26v2%3D0; _bfaStatus=success; MKT_OrderClick=ASID=4897155952&AID=4897&CSID=155952&OUID=index&CT=1655985078165&CURL=https%3A%2F%2Fm.ctrip.com%2Fhtml5%2F%3Fsid%3D155952%26allianceid%3D4897%26ouid%3Dindex&VAL={"h5_vid":"1655441049907.gadsg"}; librauuid=; _RF1=183.253.27.72; mktDpLinkSource=ullink; _pd=%7B%22r%22%3A5%2C%22d%22%3A145%2C%22_d%22%3A140%2C%22p%22%3A146%2C%22_p%22%3A1%2C%22o%22%3A149%2C%22_o%22%3A3%2C%22s%22%3A155%2C%22_s%22%3A6%7D; _bfa=1.1655441049907.gadsg.1.1655977855790.1656049564258.32.291.214059; _ubtstatus=%7B%22vid%22%3A%221655441049907.gadsg%22%2C%22sid%22%3A32%2C%22pvid%22%3A291%2C%22pid%22%3A%22214059%22%7D; Union=OUID=&AllianceID=4897&SID=155952&SourceID=&createtime=1656049564&Expires=1656654364431; GUID=09031168218403899804'
}
response = requests.post(url,headers=headers, json=payload)
# print(response.json())
data = jsonpath(response.json(),'$..data')[0]
# print(data)
momos = re.findall(r'"districtId":(.*?),.*?"districtName":"(.*?)",',data,re.S)
for a,b in momos:
# print(a)
print(b)
self.payload['districtId'] = a
self.payload['index'] = 1
time.sleep(1)
self.sendUrl()
# 得到cityId方法返回的url
def sendUrl(self):
# 先取两页看看
# while True:
for i in range(1,3):
num = self.payload['index']
print(f'第{num}页')
time.sleep(1)
self.payload['index']=num + 1
try:
response = requests.post(self.url, headers=self.headers, json=self.payload,proxies=self.ip)
# response = requests.post(self.url, headers=self.headers, json=self.payload)
# 调用方法
self.jieXi(response.json())
except:
# 设置递归深度
sys.setrecursionlimit(5)
return self.sendUrl()
# 通过第一次解析得到的详情页url---去请求详情页 请求详情页链接主要是为了得到 poild
# 评分 等级 url 通过详情页url解析得到
# 这个方法是去请求含有 评分和景区等级的url 通过 self.jieXi（） 得到链接，然后用正则，得到的poild
def ziUrl(self,poild):
try:
it = self.payload['districtId']
url = "https://m.ctrip.com/restapi/soa2/18254/json/GetSightOverview"
payload = json.dumps({
"useSightExtend": True,
"districtId": it,
"scene": "basic",
"head": {
"cid": "09031137218199825303",
"ctok": "",
"cver": "1.0",
"lang": "01",
"sid": "8888",
"syscode": "09",
"xsid": "",
"extension": []
},
"poiId": poild
})
headers = {
'authority': 'm.ctrip.com',
'cookieorigin': 'https://m.ctrip.com',
'user-agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Mobile Safari/537.36',
'content-type': 'application/json',
'accept': '*/*',
'origin': 'https://m.ctrip.com',
'sec-fetch-site': 'same-origin',
'sec-fetch-mode': 'cors',
'sec-fetch-dest': 'empty',
'referer': 'https://m.ctrip.com/webapp/you/gspoi/sight/2/0.html?poiId=75627&seo=0&isHideNavBar=YES',
'accept-language': 'zh-CN,zh;q=0.9',
'cookie': 'GUID=09031137218199825303; MKT_CKID=1655441050631.y3cpg.1v7v; _RSG=.mK7Vo8KIkFkJFbXO64ek8; _RGUID=fb92508c-9f1d-4c3e-8ba8-10662194f4ee; _RDG=2894d2e5596bd8261b1bd4f0d138733ac9; _bfaStatusPVSend=1; nfes_isSupportWebP=1; nfes_isSupportWebP=1; _ga=GA1.2.1595398136.1655441061; ibu_h5_lang=en; ibu_h5_local=en-us; StartCity_Pkg=PkgStartCity=25; ibulanguage=CN; ibulocale=zh_cn; cookiePricesDisplayed=CNY; _gid=GA1.2.2020737530.1655798638; Session=smartlinkcode=U130026&smartlinklanguage=zh&SmartLinkKeyWord=&SmartLinkQuary=&SmartLinkHost=; MKT_CKID_LMT=1655959369643; __zpspc=9.8.1655959369.1655959369.1%232%7Cwww.baidu.com%7C%7C%7C%7C%23; _jzqco=%7C%7C%7C%7C1655959370213%7C1.828646019.1655441050624.1655606559937.1655959369657.1655606559937.1655959369657.undefined.0.0.10.10; _RF1=183.253.26.192; appFloatCnt=1; _bfi=p1%3D102001%26p2%3D0%26v1%3D237%26v2%3D0; _bfaStatus=success; MKT_Pagesource=H5; librauuid=; Union=OUID=&AllianceID=4897&SID=130026&SourceID=&AppID=&OpenID=&exmktID=&createtime=1655963140&Expires=1656567939691; MKT_OrderClick=ASID=4897130026&AID=4897&CSID=130026&OUID=&CT=1655963139698&CURL=https%3A%2F%2Fm.ctrip.com%2Fwebapp%2Fyou%2Fgspoi%2Fsight%2F2.html%3Fseo%3D0%26allianceid%3D4897%26sid%3D130026%26isHideNavBar%3DYES%26from%3Dhttps%253A%252F%252Fm.ctrip.com%252Fwebapp%252Fyou%252Fgsdestination%252Fplace%252F2.html%253Fseo%253D0%2526ishideheader%253Dtrue%2526secondwakeup%253Dtrue%2526dpclickjump%253Dtrue%2526allianceid%253D4897%2526sid%253D130026%2526from%253Dhttps%25253A%25252F%25252Fm.ctrip.com%25252Fhtml5%25252F&VAL={"h5_vid":"1655441049907.gadsg"}; hotelhst=1164390341; _pd=%7B%22r%22%3A18%2C%22d%22%3A289%2C%22_d%22%3A271%2C%22p%22%3A290%2C%22_p%22%3A1%2C%22o%22%3A292%2C%22_o%22%3A2%2C%22s%22%3A293%2C%22_s%22%3A1%7D; _bfa=1.1655441049907.gadsg.1.1655886903531.1655963157778.24.250.214070; _ubtstatus=%7B%22vid%22%3A%221655441049907.gadsg%22%2C%22sid%22%3A24%2C%22pvid%22%3A250%2C%22pid%22%3A214070%7D; GUID=09031168218403899804'
}
# response = requests.post(url, headers=headers, data=payload)
response = requests.post(url, headers=headers, data=payload,proxies=self.ip)
# print(response.json())
return response.json()
# 返回子页面数据
except:
sys.setrecursionlimit(5)
return self.ziUrl(poild)
# 价格url 这个也是一样的 跟self.ziUrl（）一个道理
def priceUrl(self,poild):
try:
it = self.payload['districtId'] # 这个意思是 如果我请求的是北京，那么我下面请求的城市id也必须是北京，要保持一致
url = "https://m.ctrip.com/restapi/soa2/18254/json/getSightExtendInfo"
payload = json.dumps({
"districtId": it,
"scene": "basic",
"head": {
"cid": "09031137218199825303",
"ctok": "",
"cver": "1.0",
"lang": "01",
"sid": "8888",
"syscode": "09",
"xsid": "",
"extension": []
},
"poiId": poild
})
headers = {
'authority': 'm.ctrip.com',
'cookieorigin': 'https://m.ctrip.com',
'user-agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Mobile Safari/537.36',
'content-type': 'application/json',
'accept': '*/*',
'origin': 'https://m.ctrip.com',
'sec-fetch-site': 'same-origin',
'sec-fetch-mode': 'cors',
'sec-fetch-dest': 'empty',
'referer': 'https://m.ctrip.com/webapp/you/gspoi/sight/2/0.html?poiId=81728&seo=0&isHideNavBar=YES',
'accept-language': 'zh-CN,zh;q=0.9',
'cookie': 'GUID=09031137218199825303; MKT_CKID=1655441050631.y3cpg.1v7v; _RSG=.mK7Vo8KIkFkJFbXO64ek8; _RGUID=fb92508c-9f1d-4c3e-8ba8-10662194f4ee; _RDG=2894d2e5596bd8261b1bd4f0d138733ac9; _bfaStatusPVSend=1; nfes_isSupportWebP=1; nfes_isSupportWebP=1; _ga=GA1.2.1595398136.1655441061; ibu_h5_lang=en; ibu_h5_local=en-us; StartCity_Pkg=PkgStartCity=25; ibulanguage=CN; ibulocale=zh_cn; cookiePricesDisplayed=CNY; _gid=GA1.2.2020737530.1655798638; Session=smartlinkcode=U130026&smartlinklanguage=zh&SmartLinkKeyWord=&SmartLinkQuary=&SmartLinkHost=; MKT_CKID_LMT=1655959369643; __zpspc=9.8.1655959369.1655959369.1%232%7Cwww.baidu.com%7C%7C%7C%7C%23; _jzqco=%7C%7C%7C%7C1655959370213%7C1.828646019.1655441050624.1655606559937.1655959369657.1655606559937.1655959369657.undefined.0.0.10.10; _RF1=183.253.26.192; appFloatCnt=1; _bfi=p1%3D102001%26p2%3D0%26v1%3D237%26v2%3D0; _bfaStatus=success; MKT_Pagesource=H5; Union=OUID=&AllianceID=4897&SID=130026&SourceID=&AppID=&OpenID=&exmktID=&createtime=1655963140&Expires=1656567939691; MKT_OrderClick=ASID=4897130026&AID=4897&CSID=130026&OUID=&CT=1655963139698&CURL=https%3A%2F%2Fm.ctrip.com%2Fwebapp%2Fyou%2Fgspoi%2Fsight%2F2.html%3Fseo%3D0%26allianceid%3D4897%26sid%3D130026%26isHideNavBar%3DYES%26from%3Dhttps%253A%252F%252Fm.ctrip.com%252Fwebapp%252Fyou%252Fgsdestination%252Fplace%252F2.html%253Fseo%253D0%2526ishideheader%253Dtrue%2526secondwakeup%253Dtrue%2526dpclickjump%253Dtrue%2526allianceid%253D4897%2526sid%253D130026%2526from%253Dhttps%25253A%25252F%25252Fm.ctrip.com%25252Fhtml5%25252F&VAL={"h5_vid":"1655441049907.gadsg"}; librauuid=; hotelhst=1164390341; _pd=%7B%22r%22%3A14%2C%22d%22%3A327%2C%22_d%22%3A313%2C%22p%22%3A328%2C%22_p%22%3A1%2C%22o%22%3A330%2C%22_o%22%3A2%2C%22s%22%3A331%2C%22_s%22%3A1%7D; _bfa=1.1655441049907.gadsg.1.1655886903531.1655966576567.25.253.214070; _ubtstatus=%7B%22vid%22%3A%221655441049907.gadsg%22%2C%22sid%22%3A25%2C%22pvid%22%3A253%2C%22pid%22%3A214070%7D; GUID=09031168218403899804'
}
# response = requests.post(url, headers=headers, data=payload)
response = requests.post(url, headers=headers, data=payload,proxies=self.ip)
return response.json()
except:
sys.setrecursionlimit(5)
return self.priceUrl(poild)
# 解析景点内容
def jieXi(self,data):# 解析景点名 链接
time.sleep(self.randomtime)
try:
poiName = jsonpath(data,'$..poiName')
# 距市中心
distanceStr = jsonpath(data,'$..distanceStr')
# xx地区
displayField = jsonpath(data,'$..displayField')
detailUrl = jsonpath(data,'$..detailUrl')
for poiNames,detailUrls,distanceStrs,displayFields in zip(poiName,detailUrl,distanceStr,displayField):
# print(poiNames)
# print(detailUrls)
# 请求详情页内部 抓取相关 评分 等级
# 这个正则用来匹配 poild
poild = re.findall(r'https.*?poiId=(.*?)&', str(detailUrls), re.S)[0]
# 这个方法是返回所有的评分 景区等级
bbb = self.ziUrl(poild)
# time.sleep(self.randomtime)
# 下面这些个异常处理是因为有的景区没有评分，或是没有等级
# 没有评分，没有等级返回就为 False
# 所以要手动更改为 '无’
try: # 评分
commentScore = jsonpath(bbb,'$..commentScore')[0]
if commentScore == False:
commentScore = '无'
except:
commentScore = '无'
try: # 等级
poiLevel = jsonpath(bbb,'$..poiLevel')[0]
poiLevel = str(poiLevel)+'A'
if poiLevel == False:
poiLevel = '无'
except:
poiLevel = '无'
ccc = self.priceUrl(poild)
# time.sleep(self.randomtime)
# 解析价格
try:
price = jsonpath(ccc,'$..price')[0]
if price==False:
price = '免费'
except:
price = '免费'
print(poiNames,commentScore,distanceStrs,displayFields,poiLevel,price,detailUrls)
# 保存
self.save(poiNames,commentScore,distanceStrs,displayFields,poiLevel,price,detailUrls)
except:
# 正常退出
print('阿巴阿巴阿巴阿巴阿巴阿巴阿巴')
return self.jieXi(data)
# 保存内容到数据库
def save(self,a,b,c,d,e,f,g):
sql = 'insert into pl(poiName,commentScore,distanceStr,displayField,sightLevelStr,price,detailUrl) values(%s,%s,%s,%s,%s,%s,%s)'
val = (a,b,c,d,e,f,g)
self.mysql_cursor.execute(sql,val)
self.conn.commit()
# 执行代码
def start(self):
# 26个字母以参数的形式进行传入 # 字母 I O V 没有景点 这又是一个大坑
# 修改列表a即可
self.cityID()
if __name__ == '__main__':
aaa = XCspider()
# aaa.start()
aaa.cityID() |
需求完成
只是请求太过频繁，需要调节请求速度
博主速度不会调
懂行的小伙伴，可以在这里留言，我请你吃棒棒糖，双享棒一块钱的哦（5毛的不吃）
扫一扫
966
暂无认证
到【灌水乐园】发言
CSDN-Ada助手: 推荐 网络 技能树：https://edu.csdn.net/skill/network?utm_source=AI_act_network