I've spend a lot of time trying to set up aws secrets manager with jdbc kafka connector in docker compose locally, but a I didn' succeed. I've tested other parts separately and that's ok, except aws secrets manager. In this context three files are used. (init-secrets-manager.sh | test-jdbc-source-connector.json | docker-compose.yml)
The key is created successfully!
aws-localstack | {
aws-localstack | "ARN": "arn:aws:secretsmanager:us-east-1:000000000000:secret:bd-test-FRhcf",
aws-localstack | "Name": "bd-test",
aws-localstack | "VersionId": "5cb8132b-cf25-44d0-ae81-bd5570ef01aa"
aws-localstack | }
I'm getting this error when set up a connector: (connect-distributed.log log of the landoop image)
Caused by: com.amazonaws.services.secretsmanager.model.AWSSecretsManagerException: The security token included in the request is invalid. (Service: AWSSecretsManager; Status Code: 400; Error Code: UnrecognizedClientException; Request ID: 16b70666-0536-49d6-aa32-1d5bf21b615d)
init-secrets-manager.sh
aws --endpoint http://localhost:4566 secretsmanager create-secret --name bd-test --description "Some secret" --secret-string '{"user":"adm","password":"adm","decryptionkey":"123"}' --tags '[{"Key":"user", "Value":"adm"},{"Key":"password","Value":"adm"},{"Key":"decryptionkey","Value":"123"}]' --region us-east-1
test-jdbc-source-connector.json
{
"name": "test-jdbc-source-connector",
"config": {
"name": "test-jdbc-source-connector",
"connector.class": "io.confluent.connect.jdbc.JdbcSourceConnector",
"tasks.max": "1",
"poll.interval.ms": "10000",
"timestamp.delay.interval.ms": "80000",
"batch.max.rows": "1000",
"topic.prefix": "customer_topic_test",
"value.converter": "io.confluent.connect.avro.AvroConverter",
"value.converter.use.latest.version": "true",
"value.converter.enhanced.avro.schema.support": "true",
"value.converter.schema.registry.url": "http://localhost:8081",
"value.converter.auto.register.schemas": "true",
"value.converter.schemas.enable": "true",
"mode": "incrementing",
"incrementing.column.name": "id",
"query": "SELECT ID, convert_from(decrypt(NAME, '${aws:bd-test:decryptionkey}'::bytea,'aes'),'UTF-8') as NAME FROM CUSTOMER",
"schema.pattern": "adm",
"connection.url": "jdbc:postgresql://postgres:5432/customer?currentSchema=adm",
"connection.user": "${aws:bd-test:user}",
"connection.password": "${aws:bd-test:password}",
"connection.attempts": "5",
"transforms": "createKey",
"transforms.createKey.type":"org.apache.kafka.connect.transforms.ValueToKey",
"transforms.createKey.fields":"id"
}
}
docker-compose.yml
version: '3.3'
services:
aws-localstack:
container_name: aws-localstack
hostname: aws-localstack
image: localstack/localstack
ports:
- 4566:4566
- 8080:8080
environment:
- SERVICES=secretsmanager
- DEBUG=1
- DATA_DIR=/tmp/localstack/data
- DOCKER_HOST=unix:///var/run/docker.sock
- DEFAULT_REGION=us-east-1
- AWS_ACCESS_KEY_ID=test
- AWS_SECRET_ACCESS_KEY=test
- LS_LOG=trace
volumes:
- ./localstack:/tmp/localstack
- /var/run/docker.sock:/var/run/docker.sock
- ./localstack/init-secrets-manager.sh:/docker-entrypoint-initaws.d/init-secrets-manager.sh
networks:
- network_test
postgresql:
image: postgres:11.5-alpine
container_name: postgres
hostname: postgres
ports:
- 5432:5432
environment:
POSTGRES_USER: adm
POSTGRES_PASSWORD: adm
POSTGRES_DB: customer
PGDATA: /var/lib/postgresql/data/pgdata
volumes:
- test_postgresql_data:/var/lib/postgresql/data
- ./postgresql/init.sh:/docker-entrypoint-initdb.d/init.sh
networks:
- network_test
kafka:
container_name: kafka
hostname: kafka
image: landoop/fast-data-dev
ports:
- 9092:9092
- 8081:8081
- 8082:8082
- 8083:8083
- 2181:2181
- 3030:3030
- 9581:9581
- 9582:9582
- 9583:9583
- 9584:9584
environment:
- CONNECT_CONFIG_PROVIDERS=aws
- CONNECT_CONFIG_PROVIDERS_AWS_CLASS=io.lenses.connect.secrets.providers.AWSSecretProvider
- CONNECT_CONFIG_PROVIDERS_AWS_PARAM_AWS_ACCESS_KEY=test
- CONNECT_CONFIG_PROVIDERS_AWS_PARAM_AWS_AUTH_METHOD=credentialst
- CONNECT_CONFIG_PROVIDERS_AWS_PARAM_AWS_REGION=us-east-1
- CONNECT_CONFIG_PROVIDERS_AWS_PARAM_AWS_SECRET_KEY=test
- CONNECT_INTERNAL_KEY_CONVERTER=org.apache.kafka.connect.json.JsonConverter
- CONNECT_INTERNAL_VALUE_CONVERTER=org.apache.kafka.connect.json.JsonConverter
volumes:
- ./localstack:/tmp/localstack
networks:
- network_test
volumes:
test_postgresql_data:
networks:
network_test:
driver: bridge