I’m having issues running part of my code synchronously when using AsyncHTMLSession.
Need the following:
- logging with AsyncHTMLSession session (synchronously)
- retrieve responses to numerous links using a function (asynchronously)
- rendering responses to all html’s since site is dynamic (asynchronously)
- parsing all data (could be synchronously)
here’s the code:
def get_session_tasks(session):
tasks = []
for url in urls:
tasks.append(session.get(url))
return tasks
def get_render_tasks(responses):
tasks = []
for response in responses:
tasks.append(response.html.arender(sleep=1))
return tasks
async def main():
with AsyncHTMLSession() as session:
session_tasks = get_session_tasks(session)
responses = await asyncio.gather(*session_tasks)
render_tasks = get_render_tasks(responses)
rendered_responses = await asyncio.gather(*render_tasks)
return rendered_responses
rendered_responses = asyncio.run(main())
parse(rendered_responses) # parse is a synchronous function