How to mock the boto3 client session requests for secretsmanager to either return a specific response or raise an exception

user_1356423 0
Viewed 5378

Given code like:

def do_stuff():

    session = boto3.session.Session()
    client = session.client(service_name="secretsmanager", region_name=region_name)
    client.get_secret_value(SecretId=sendgrid_api_key_arn)

how do I mock client.get_secret_value("some-value") to return some value

and how do I mock it to throw an exception

@patch("boto3.session")
def test_get_sendgrid_api_key_secret_when_client_error(mock_session):
        session = mock_session.Session();
        client = session.client()
        client.get_secret_value().return_value = 
                 {"SecretString": "my-secret"} <- this is wrapped in a MagicMock which is useless.
pytest-mock boto3 pytest python
3 Answers

You'll need to patch client.get_secret_value, by adding one more decorator. Once this is done you an set the return_value to exception you need.

@patch("boto3.session")
@patch("boto3.session.Session.client.get_secret_value")
def test_get_sendgrid_api_key_secret_when_client_error(mock_session,second_fn):
        with pytest.raises('CustomException')
        session = mock_session.Session();
        second_fn.return_value = lambda : Exception('CustomException')
        client = session.client()
        client.get_secret_value()

When client.get_secret_value() is executed, test case will pass for raising the intended exception. Similarly instead of an lambda we can assign any other value to mock the return of function.

user_159062910

You need to set a tree of return values:

  • boto3.session.Session needs to return a mock object
  • That mock object needs a client method that returns another mock object
  • And that mock object needs a get_secret_value method that returns a fake value

If I assume that target.py exists and contains:

import boto3.session


def do_stuff():
    session = boto3.session.Session()
    client = session.client(service_name="secretsmanager", region_name='myregion')
    return client.get_secret_value(SecretId='some-secret-id')

Then I can test it like this:

from unittest import mock

import target


@mock.patch("boto3.session.Session")
def test_do_stuff(mock_session_class):
    mock_session_object = mock.Mock()
    mock_client = mock.Mock()
    mock_client.get_secret_value.return_value = {'SecretString': 'my-secret'}
    mock_session_object.client.return_value = mock_client
    mock_session_class.return_value = mock_session_object

    res = target.do_stuff()
    assert res['SecretString'] == 'my-secret'

This works, although I suspect there's a more graceful way to set it up.

user_1473560

One problem with patching boto3.session manually is that if you are using other AWS services aside from secretsmanager, then the mocked patch would be applied to everything, and this may not be what you want.

I propose here 2 solutions that are dedicated for patching/wrapping the secretsmanager functionality:

  1. See ./test_secret_manual_amend.py below. Intercept the botocore.client.BaseClient._make_api_call.GetSecretValue and control how the response would be. This is the more flexible solution as you can freely check the accessed secret-name, raise exceptions, and return anything you like.
  2. See ./test_secret_using_moto.py below. Use moto.mock_secretsmanager. This is the cleanest approach since this imitates how you would normally interact with the actual AWs secretsmanager. But probably a bit less flexible than solution-1.

./src.py

import boto3

sendgrid_api_key_arn = "some_name"
region_name = "ap-southeast-1"
credentials = {}


def do_stuff():
    session = boto3.session.Session(**credentials)
    client = session.client(service_name="secretsmanager", region_name=region_name)
    return client.get_secret_value(SecretId=sendgrid_api_key_arn)

./test_secret_manual_amend.py

import botocore
import json
import pytest

from src import do_stuff, sendgrid_api_key_arn


def _amend_get_secret_value(secret_name, secret_value, mocker):
    orig = botocore.client.BaseClient._make_api_call

    def amend_make_api_call(self, operation_name, kwargs):
        # Intercept boto3 operations for <secretsmanager.get_secret_value>. Optionally, you can also
        # check on the argument <SecretId> and control how you want the response would be. This is
        # a very flexible solution as you have full control over the whole process of fetching a
        # secret.
        if operation_name == 'GetSecretValue' and kwargs["SecretId"] == secret_name:
            if isinstance(secret_value, Exception):
                raise secret_value
            return {
                'Name': secret_name,
                'SecretString': secret_value,
            }

        return orig(self, operation_name, kwargs)

    mocker.patch('botocore.client.BaseClient._make_api_call', new=amend_make_api_call)


@pytest.mark.parametrize(
    'secret_value',
    [
        "some value",
        str(1993),
        json.dumps({"SecretString": "my-secret"}),
        json.dumps([2, 3, 5, 7, 11, 13, 17, 19]),
        KeyError("How dare you touch my secret!"),
        ValueError("Oh my goodness you even have the guts to repeat it!!!"),
    ],
)
def test_secret_manual_amend(secret_value, mocker):
    _amend_get_secret_value(sendgrid_api_key_arn, secret_value, mocker)

    if isinstance(secret_value, Exception):
        with pytest.raises(type(secret_value)) as error:
            do_stuff()
        result = error
    else:
        result = do_stuff()

    print("Result:", result)

./test_secret_using_moto.py

import boto3
import json
from moto import mock_secretsmanager
import pytest

from src import do_stuff, sendgrid_api_key_arn


def _setup_secrets_manager(secret_name, secret_value):
    secret_manager = boto3.client('secretsmanager')
    secret_manager.create_secret(
        Name=secret_name,
        SecretString=secret_value,
    )


@mock_secretsmanager
@pytest.mark.parametrize(
    'secret_value',
    [
        "some value",
        str(1993),
        json.dumps({"SecretString": "my-secret"}),
        json.dumps([2, 3, 5, 7, 11, 13, 17, 19]),
    ],
)
def test_secret_using_moto(secret_value):
    _setup_secrets_manager(sendgrid_api_key_arn, secret_value)

    result = do_stuff()

    print("Result:", result)

Output:

$ pytest -rP
====================================================================================== test session starts ======================================================================================
test_secret_manual_amend.py ......                                                                                                                                                        [ 60%]
test_secret_using_moto.py ....                                                                                                                                                            [100%]

============================================================================================ PASSES =============================================================================================


_____________________________________________________________________________ test_secret_manual_amend[some value] ______________________________________________________________________________
------------------------------------------------------------------------------------- Captured stdout call --------------------------------------------------------------------------------------
Result: {'Name': 'some_name', 'SecretString': 'some value'}
________________________________________________________________________________ test_secret_manual_amend[1993] _________________________________________________________________________________
------------------------------------------------------------------------------------- Captured stdout call --------------------------------------------------------------------------------------
Result: {'Name': 'some_name', 'SecretString': '1993'}
____________________________________________________________________ test_secret_manual_amend[{"SecretString": "my-secret"}] ____________________________________________________________________
------------------------------------------------------------------------------------- Captured stdout call --------------------------------------------------------------------------------------
Result: {'Name': 'some_name', 'SecretString': '{"SecretString": "my-secret"}'}
____________________________________________________________________ test_secret_manual_amend[[2, 3, 5, 7, 11, 13, 17, 19]] _____________________________________________________________________
------------------------------------------------------------------------------------- Captured stdout call --------------------------------------------------------------------------------------
Result: {'Name': 'some_name', 'SecretString': '[2, 3, 5, 7, 11, 13, 17, 19]'}
____________________________________________________________________________ test_secret_manual_amend[secret_value4] ____________________________________________________________________________
------------------------------------------------------------------------------------- Captured stdout call --------------------------------------------------------------------------------------
Result: <ExceptionInfo KeyError('How dare you touch my secret!') tblen=4>
____________________________________________________________________________ test_secret_manual_amend[secret_value5] ____________________________________________________________________________
------------------------------------------------------------------------------------- Captured stdout call --------------------------------------------------------------------------------------
Result: <ExceptionInfo ValueError('Oh my goodness you even have the guts to repeat it!!!') tblen=4>


______________________________________________________________________________ test_secret_using_moto[some value] _______________________________________________________________________________
------------------------------------------------------------------------------------- Captured stdout call --------------------------------------------------------------------------------------
Result: {'ARN': 'arn:aws:secretsmanager:ap-southeast-1:1234567890:secret:some_name-ghtZn', 'Name': 'some_name', 'VersionId': '42a45aad-8764-49fb-b1ad-67759063f804', 'SecretString': 'some value', 'VersionStages': ['AWSCURRENT'], 'CreatedDate': datetime.datetime(2021, 5, 13, 15, 3, 43, tzinfo=tzlocal()), 'ResponseMetadata': {'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'amazon.com'}, 'RetryAttempts': 0}}
_________________________________________________________________________________ test_secret_using_moto[1993] __________________________________________________________________________________
------------------------------------------------------------------------------------- Captured stdout call --------------------------------------------------------------------------------------
Result: {'ARN': 'arn:aws:secretsmanager:ap-southeast-1:1234567890:secret:some_name-JdOXC', 'Name': 'some_name', 'VersionId': '098b2878-4368-4f25-b75d-942e82745257', 'SecretString': '1993', 'VersionStages': ['AWSCURRENT'], 'CreatedDate': datetime.datetime(2021, 5, 13, 15, 3, 44, tzinfo=tzlocal()), 'ResponseMetadata': {'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'amazon.com'}, 'RetryAttempts': 0}}
_____________________________________________________________________ test_secret_using_moto[{"SecretString": "my-secret"}] _____________________________________________________________________
------------------------------------------------------------------------------------- Captured stdout call --------------------------------------------------------------------------------------
Result: {'ARN': 'arn:aws:secretsmanager:ap-southeast-1:1234567890:secret:some_name-jQgDK', 'Name': 'some_name', 'VersionId': '04bd6ceb-6ec9-427b-817c-c90360abfcd9', 'SecretString': '{"SecretString": "my-secret"}', 'VersionStages': ['AWSCURRENT'], 'CreatedDate': datetime.datetime(2021, 5, 13, 15, 3, 44, tzinfo=tzlocal()), 'ResponseMetadata': {'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'amazon.com'}, 'RetryAttempts': 0}}
_____________________________________________________________________ test_secret_using_moto[[2, 3, 5, 7, 11, 13, 17, 19]] ______________________________________________________________________
------------------------------------------------------------------------------------- Captured stdout call --------------------------------------------------------------------------------------
Result: {'ARN': 'arn:aws:secretsmanager:ap-southeast-1:1234567890:secret:some_name-dQpLq', 'Name': 'some_name', 'VersionId': '94b7a169-c32b-4424-b47c-2dee2802c211', 'SecretString': '[2, 3, 5, 7, 11, 13, 17, 19]', 'VersionStages': ['AWSCURRENT'], 'CreatedDate': datetime.datetime(2021, 5, 13, 15, 3, 44, tzinfo=tzlocal()), 'ResponseMetadata': {'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'amazon.com'}, 'RetryAttempts': 0}}
====================================================================================== 10 passed in 1.26s =======================================================================================
user_110438250
Related