aidevelopementtoolkit.logging_utils.boto3_utils

create_s3_client()

Create a boto3 S3 client from environment variables.

Reads the following environment variables:

  • S3_ENDPOINT_URL - custom endpoint URL.
  • AWS_ACCESS_KEY_ID - access key ID.
  • AWS_SECRET_ACCESS_KEY - secret access key.
Returns:
  • S3

    Configured S3 client.

Raises:
  • EnvironmentError

    If any of the required environment variables are not set.

Examples:

>>> import os
>>> os.environ["S3_ENDPOINT_URL"] = "http://localhost:9000"
>>> os.environ["AWS_ACCESS_KEY_ID"] = "minioadmin"
>>> os.environ["AWS_SECRET_ACCESS_KEY"] = "minioadmin"
>>> client = create_s3_client()
Source code in aidevelopementtoolkit/logging_utils/boto3_utils.py
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
def create_s3_client():
    """
    Create a boto3 S3 client from environment variables.

    Reads the following environment variables:

    - `S3_ENDPOINT_URL` - custom endpoint URL.
    - `AWS_ACCESS_KEY_ID` - access key ID.
    - `AWS_SECRET_ACCESS_KEY` - secret access key.

    Returns
    -------
    botocore.client.S3
        Configured S3 client.

    Raises
    ------
    EnvironmentError
        If any of the required environment variables are not set.

    Examples
    --------
    >>> import os
    >>> os.environ["S3_ENDPOINT_URL"] = "http://localhost:9000"
    >>> os.environ["AWS_ACCESS_KEY_ID"] = "minioadmin"
    >>> os.environ["AWS_SECRET_ACCESS_KEY"] = "minioadmin"
    >>> client = create_s3_client()
    """

    required = ("S3_ENDPOINT_URL", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY")
    missing = [var for var in required if not os.environ.get(var)]

    if missing:
        logger.error(f"Missing required environment variables: {missing}")
        raise EnvironmentError()

    return boto3.client(
        "s3",
        endpoint_url=os.environ["S3_ENDPOINT_URL"],
        aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
        aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
    )

read_s3_object(client, bucket: str, key: str) -> bytes

Download an S3 object and return its content as bytes.

Displays a tqdm progress bar during the download, using the object's Content-Length as the total when available.

Parameters:
  • client (S3) –

    S3 client returned by :func:create_s3_client.

  • bucket (str) –

    Name of the S3 bucket.

  • key (str) –

    Key (path) of the object inside the bucket.

Returns:
  • bytes

    Raw content of the S3 object.

Examples:

>>> client = create_s3_client()
>>> data = read_s3_object(client, bucket="my-bucket", key="data/file.npy")
Source code in aidevelopementtoolkit/logging_utils/boto3_utils.py
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
def read_s3_object(
        client,
        bucket: str,
        key: str,
    ) -> bytes:
    """
    Download an S3 object and return its content as bytes.

    Displays a `tqdm` progress bar during the download, using the
    object's `Content-Length` as the total when available.

    Parameters
    ----------
    client : botocore.client.S3
        S3 client returned by :func:`create_s3_client`.

    bucket : str
        Name of the S3 bucket.

    key : str
        Key (path) of the object inside the bucket.

    Returns
    -------
    bytes
        Raw content of the S3 object.

    Examples
    --------
    >>> client = create_s3_client()
    >>> data = read_s3_object(client, bucket="my-bucket", key="data/file.npy")
    """

    response = client.head_object(Bucket=bucket, Key=key)
    total = response["ContentLength"]

    buffer = io.BytesIO()

    config = TransferConfig(
        multipart_threshold=_CHUNK_SIZE,
        multipart_chunksize=_CHUNK_SIZE,
    )

    with tqdm(
        total=total,
        unit="B",
        unit_scale=True,
        unit_divisor=1024,
        desc=f"Reading {os.path.basename(key)} from S3",
        colour="yellow",
        leave=False,
    ) as progress:
        client.download_fileobj(
            bucket,
            key,
            buffer,
            Config=config,
            Callback=lambda n: progress.update(n),
        )

    buffer.seek(0)
    return buffer.getvalue()

write_s3_object(client, bucket: str, key: str, data: bytes) -> None

Upload bytes to an S3 object.

Displays a tqdm progress bar during the upload.

Parameters:
  • client (S3) –

    S3 client returned by :func:create_s3_client.

  • bucket (str) –

    Name of the destination S3 bucket.

  • key (str) –

    Key (path) of the object to write inside the bucket.

  • data (bytes) –

    Raw bytes to upload.

Examples:

>>> client = create_s3_client()
>>> write_s3_object(client, bucket="my-bucket", key="data/file.npy", data=b"...")
Source code in aidevelopementtoolkit/logging_utils/boto3_utils.py
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
def write_s3_object(
        client,
        bucket: str,
        key: str,
        data: bytes,
    ) -> None:
    """
    Upload bytes to an S3 object.

    Displays a `tqdm` progress bar during the upload.

    Parameters
    ----------
    client : botocore.client.S3
        S3 client returned by :func:`create_s3_client`.

    bucket : str
        Name of the destination S3 bucket.

    key : str
        Key (path) of the object to write inside the bucket.

    data : bytes
        Raw bytes to upload.

    Examples
    --------
    >>> client = create_s3_client()
    >>> write_s3_object(client, bucket="my-bucket", key="data/file.npy", data=b"...")
    """

    total = len(data)
    buffer = io.BytesIO(data)

    with tqdm(
        total=total,
        unit="B",
        unit_scale=True,
        unit_divisor=1024,
        desc=f"Writing {os.path.basename(key)} to S3",
        colour="green",
        leave=False,
    ) as progress:

        try:
            client.upload_fileobj(buffer, bucket, key, Callback=lambda n: progress.update(n))
        except Exception as e:
            logger.error(f"The S3 upload of {key} in the bucket {bucket} as failed. \n{e}")

parse_s3_path(path: str) -> Tuple[str, str]

Parse an S3 path into bucket and key.

Parameters:
  • path (str) –

    S3 path in the format s3://bucket/key.

Returns:
  • Tuple[str, str]

    Bucket name and object key.

Source code in aidevelopementtoolkit/logging_utils/boto3_utils.py
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
def parse_s3_path(path: str) -> Tuple[str, str]:
    """Parse an S3 path into bucket and key.

    Parameters
    ----------
    path : str
        S3 path in the format `s3://bucket/key`.

    Returns
    -------
    Tuple[str, str]
        Bucket name and object key.
    """

    s3_path = path.removeprefix("s3://")

    if "/" not in s3_path:
        logger.error("Invalid S3 path. Expected format: 's3://bucket/key'.")
        raise ValueError()

    bucket, key = s3_path.split("/", maxsplit=1)

    if not bucket or not key:
        logger.error("Invalid S3 path. Expected format: 's3://bucket/key'.")
        raise ValueError()

    return bucket, key