trjhtr

I need a program that needs to gather data from large list of API end points. Below is a mock-up program that attempts to make 10000 requests as fast as possible. Any suggestions on how to improve on this (especially for speed) is highly welcome. Experimentation showed that the Semaphore cap of around 100 gave the best speed.

import asyncio
from aiohttp import ClientSession
import datetime
import time
import sys

def processData(data):
 time.sleep(0.001)
 return data

async def fetch(url, session):
 async with session.get(url) as response:
 data = await response.read()
 data = processData(data)
 return data

async def bound_fetch(sem, url, session):
 async with sem:
 return await fetch(url, session)


async def run(loop,N):
 url = "https://www.example.com"
 tasks = 
 sem = asyncio.Semaphore(100)
 async with ClientSession() as session:
 for i in range(N):
 task = loop.create_task(bound_fetch(sem, url, session))
 tasks.append(task)

 print("Done starting tasks".format(N))
 starttime = time.time()
 print(datetime.datetime.now())
 responses = await asyncio.gather(*tasks)
 print("Done completing tasks in: ".format(N,time.time()-starttime))

 return responses



args = sys.argv
loop = asyncio.get_event_loop()

if __name__ == "__main__":
 if len(sys.argv) == 2:
 N = int(sys.argv[1])
 else:
 N = 10000
 maintask = loop.create_task(run(loop, N))
 result = loop.run_until_complete(maintask)
 print(len(result))