Download an S3 object and return its content as bytes.
Displays a tqdm progress bar during the download, using the
object's Content-Length as the total when available.
| Parameters: |
-
client
(S3)
–
S3 client returned by :func:create_s3_client.
-
bucket
(str)
–
-
key
(str)
–
Key (path) of the object inside the bucket.
|
| Returns: |
-
bytes
–
Raw content of the S3 object.
|
Examples:
>>> client = create_s3_client()
>>> data = read_s3_object(client, bucket="my-bucket", key="data/file.npy")
Source code in aidevelopementtoolkit/logging_utils/boto3_utils.py
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122 | def read_s3_object(
client,
bucket: str,
key: str,
) -> bytes:
"""
Download an S3 object and return its content as bytes.
Displays a `tqdm` progress bar during the download, using the
object's `Content-Length` as the total when available.
Parameters
----------
client : botocore.client.S3
S3 client returned by :func:`create_s3_client`.
bucket : str
Name of the S3 bucket.
key : str
Key (path) of the object inside the bucket.
Returns
-------
bytes
Raw content of the S3 object.
Examples
--------
>>> client = create_s3_client()
>>> data = read_s3_object(client, bucket="my-bucket", key="data/file.npy")
"""
response = client.head_object(Bucket=bucket, Key=key)
total = response["ContentLength"]
buffer = io.BytesIO()
config = TransferConfig(
multipart_threshold=_CHUNK_SIZE,
multipart_chunksize=_CHUNK_SIZE,
)
with tqdm(
total=total,
unit="B",
unit_scale=True,
unit_divisor=1024,
desc=f"Reading {os.path.basename(key)} from S3",
colour="yellow",
leave=False,
) as progress:
client.download_fileobj(
bucket,
key,
buffer,
Config=config,
Callback=lambda n: progress.update(n),
)
buffer.seek(0)
return buffer.getvalue()
|