Performance comparison with boto3 #359
Description
Hey, sorry if this was asked already. I ran a simple test comparing aioboto3 calls to boto3 calls wrapped in asyncio.to_thread. I compared execution time of s3.get_object on a small file (~100kb) from within AWS network, to minimize network latency. I'm adding the code of the comparison at the end. The results I got are:
aioboto3: 10 calls took 0.16 seconds.
boto3: 10 calls took 0.10 seconds.
aioboto3: 100 calls took 0.45 seconds.
boto3: 100 calls took 0.30 seconds.
aioboto3: 1000 calls took 2.78 seconds.
boto3: 1000 calls took 2.89 seconds.
aioboto3: 10000 calls took 27.65 seconds.
boto3: 10000 calls took 28.58 seconds.
I'm wondering, am I doing something wrong, or is it expected for the aioboto calls to take more for smaller lists?
Thanks in advance for your help!
import asyncio
import time
import aioboto3
import boto3
BUCKET = "..."
KEY = "..."
NUM_CALLS = [10, 100, 1000, 10000]
boto3_client = boto3.client("s3")
async def aioboto3_get_object(s3_client, bucket, key):
response = await s3_client.get_object(Bucket=bucket, Key=key)
data = await response["Body"].read()
return data
async def run_aioboto3_calls(num_calls):
session = aioboto3.Session()
async with session.client("s3") as s3_client:
tasks = [
asyncio.create_task(aioboto3_get_object(s3_client, BUCKET, KEY))
for _ in range(num_calls)
]
results = await asyncio.gather(*tasks)
return results
def boto3_get_object(bucket, key):
response = boto3_client.get_object(Bucket=bucket, Key=key)
data = response["Body"].read()
return data
async def run_boto3_calls(num_calls):
tasks = [
asyncio.to_thread(boto3_get_object, BUCKET, KEY)
for _ in range(num_calls)
]
results = await asyncio.gather(*tasks)
return results
async def main():
print("Starting performance comparison for 1000 S3 get_object calls...\n")
for num_calls in NUM_CALLS:
# Time the aioboto3 version.
start = time.perf_counter()
await run_aioboto3_calls(num_calls)
aioboto3_duration = time.perf_counter() - start
print(f"aioboto3: {num_calls} calls took {aioboto3_duration:.2f} seconds.")
# Time the boto3 version (wrapped in asyncio.to_thread).
start = time.perf_counter()
await run_boto3_calls(num_calls)
boto3_duration = time.perf_counter() - start
print(f"boto3: {num_calls} calls took {boto3_duration:.2f} seconds.")
if __name__ == "__main__":
asyncio.run(main())