生意参谋-市场大盘,市场排行爬虫脚本,采用pyppeteer+PIL方式处理数值图片问题,人工登录,并将爬取结果保存为csv文件
脚本截取:
import asyncio
from pyppeteer import launch
from PIL import Image
import pytesseract
import datetime
import time
import pandas as pd
async def main():
start_time = datetime.datetime.now()
print('浏览器初始化……')
browser = await launch(headless=False, args=['--disable-infobars',f'--window-size=1366,768'])
page = await browser.newPage()
await page.setViewport({'width': 1366, 'height': 768})
await page.goto('https://login.taobao.com/member/login.jhtml?redirectURL=https://www.taobao.com/') #从淘宝登录
await page.evaluate(
'''() =>{ Object.defineProperties(navigator,{ webdriver:{ get: () => false } }) }''')
await asyncio.sleep(30) #人工输入账号密码登录
print('开始登录……')
await page.goto('https://sycm.taobao.com/mc/mq/overview?cateFlag=2&cateId=50022270&dateType=week') #登录后跳转到生意参谋
await asyncio.sleep(10)
#await page.click('#content-container > div.ebase-metaDecorator__root > div > div.ebase-FaCommonFilter__root > div > div > div > div > div.ebase-FaCommonFilter__top > div.ebase-FaCommonFilter__right > div > div > div.oui-date-picker-particle-button > button.ant-btn.oui-canary-btn.ant-btn-primary.ant-btn-sm > span')
await page.evaluate(
'''() =>{ Object.defineProperties(navigator,{ webdriver:{ get: () => false } }) }''')
#uvHits = await page.waitForSelector('#sycm-mc-mq-cate-trend > div.oui-card-content.alife-one-design-sycm-indexes-trend-card-content > div > div.alife-one-design-sycm-indexes-trend-index-container > div > div > div:nth-child(1) > div.alife-one-design-sycm-indexes-trend-index-item-multiple-line-selectable.alife-one-design-sycm-indexes-trend-index-item-uvHits.active > div > div.oui-index-cell-indexValue.oui-num > div > div:nth-child(2) > img')
#await uvHits.screenshot({'path': 'uvHits.png'})
#await asyncio.sleep(3)
print('开始爬取市场大盘……')