爬虫坑总结

import aiohttp
import asyncio
import pickle
import pyquery as pq
qid=None
with open('qid','rb') as f:
    qid=pickle.load(f)
print(len(qid))
it=iter(qid)
result=[]
async def (client,it,r):
    while True:
        try:
            text,answerid=next(it)
        except StopIteration:
            break
        headers = {'X-Requested-With': 'XMLHttpRequest','User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.89 Safari/537.36'}
        href='http://gzls.cooco.net.cn/answerdetail/'+answerid
        async with client.post(href,headers=headers) as resp:
            htmltext= await resp.text()
            r.append((text,htmltext))
            print(len(htmltext))
            await asyncio.sleep(1)
    return
with aiohttp.ClientSession() as client:
    coroutine=[asyncio.ensure_future(fetch(client,it,result)) for i in range(3)]
    html = asyncio.get_event_loop().run_until_complete(asyncio.wait(coroutine))
print(len(result),len(qid))
with open('qapickle4','wb') as f:
    pickle.dump(result,f)
#协程函数里出错要看jupyter notebook控制台