How to set up aws secrets manager with jdbc kafka connector in docker compose locally?

Viewed 546

I've spend a lot of time trying to set up aws secrets manager with jdbc kafka connector in docker compose locally, but a I didn' succeed. I've tested other parts separately and that's ok, except aws secrets manager. In this context three files are used. (init-secrets-manager.sh | test-jdbc-source-connector.json | docker-compose.yml)

The key is created successfully!

aws-localstack    | {
aws-localstack    |     "ARN": "arn:aws:secretsmanager:us-east-1:000000000000:secret:bd-test-FRhcf",
aws-localstack    |     "Name": "bd-test",
aws-localstack    |     "VersionId": "5cb8132b-cf25-44d0-ae81-bd5570ef01aa"
aws-localstack    | }

I'm getting this error when set up a connector: (connect-distributed.log log of the landoop image)

Caused by: com.amazonaws.services.secretsmanager.model.AWSSecretsManagerException: The security token included in the request is invalid. (Service: AWSSecretsManager; Status Code: 400; Error Code: UnrecognizedClientException; Request ID: 16b70666-0536-49d6-aa32-1d5bf21b615d)

init-secrets-manager.sh

aws --endpoint http://localhost:4566 secretsmanager create-secret --name bd-test --description "Some secret" --secret-string '{"user":"adm","password":"adm","decryptionkey":"123"}' --tags '[{"Key":"user", "Value":"adm"},{"Key":"password","Value":"adm"},{"Key":"decryptionkey","Value":"123"}]' --region us-east-1

test-jdbc-source-connector.json

{
  "name": "test-jdbc-source-connector",
  "config": {
    "name": "test-jdbc-source-connector",
    "connector.class": "io.confluent.connect.jdbc.JdbcSourceConnector",
    "tasks.max": "1",
    "poll.interval.ms": "10000",
    "timestamp.delay.interval.ms": "80000",
    "batch.max.rows": "1000",
    "topic.prefix": "customer_topic_test",
    "value.converter": "io.confluent.connect.avro.AvroConverter",
    "value.converter.use.latest.version": "true",
    "value.converter.enhanced.avro.schema.support": "true",
    "value.converter.schema.registry.url": "http://localhost:8081",
    "value.converter.auto.register.schemas": "true",
    "value.converter.schemas.enable": "true",
    "mode": "incrementing",
    "incrementing.column.name": "id",
    "query": "SELECT ID, convert_from(decrypt(NAME, '${aws:bd-test:decryptionkey}'::bytea,'aes'),'UTF-8') as NAME FROM CUSTOMER",
    "schema.pattern": "adm",
    "connection.url": "jdbc:postgresql://postgres:5432/customer?currentSchema=adm",
    "connection.user": "${aws:bd-test:user}",
    "connection.password": "${aws:bd-test:password}",
    "connection.attempts": "5",
    "transforms": "createKey",
    "transforms.createKey.type":"org.apache.kafka.connect.transforms.ValueToKey",
    "transforms.createKey.fields":"id"
  }
}

docker-compose.yml

version: '3.3'

services:

  aws-localstack:
    container_name: aws-localstack
    hostname: aws-localstack
    image: localstack/localstack
    ports:
      - 4566:4566
      - 8080:8080
    environment:
      - SERVICES=secretsmanager
      - DEBUG=1
      - DATA_DIR=/tmp/localstack/data
      - DOCKER_HOST=unix:///var/run/docker.sock
      - DEFAULT_REGION=us-east-1
      - AWS_ACCESS_KEY_ID=test
      - AWS_SECRET_ACCESS_KEY=test
      - LS_LOG=trace
    volumes:
      - ./localstack:/tmp/localstack
      - /var/run/docker.sock:/var/run/docker.sock
      - ./localstack/init-secrets-manager.sh:/docker-entrypoint-initaws.d/init-secrets-manager.sh
    networks:
      - network_test

  postgresql:
    image: postgres:11.5-alpine
    container_name: postgres
    hostname: postgres
    ports:
      - 5432:5432
    environment:
      POSTGRES_USER: adm
      POSTGRES_PASSWORD: adm
      POSTGRES_DB: customer
      PGDATA: /var/lib/postgresql/data/pgdata
    volumes:
      - test_postgresql_data:/var/lib/postgresql/data
      - ./postgresql/init.sh:/docker-entrypoint-initdb.d/init.sh
    networks:
      - network_test

  kafka:
    container_name: kafka
    hostname: kafka
    image: landoop/fast-data-dev
    ports:
      - 9092:9092
      - 8081:8081
      - 8082:8082
      - 8083:8083
      - 2181:2181
      - 3030:3030
      - 9581:9581
      - 9582:9582
      - 9583:9583
      - 9584:9584
    environment:
      - CONNECT_CONFIG_PROVIDERS=aws
      - CONNECT_CONFIG_PROVIDERS_AWS_CLASS=io.lenses.connect.secrets.providers.AWSSecretProvider
      - CONNECT_CONFIG_PROVIDERS_AWS_PARAM_AWS_ACCESS_KEY=test
      - CONNECT_CONFIG_PROVIDERS_AWS_PARAM_AWS_AUTH_METHOD=credentialst
      - CONNECT_CONFIG_PROVIDERS_AWS_PARAM_AWS_REGION=us-east-1
      - CONNECT_CONFIG_PROVIDERS_AWS_PARAM_AWS_SECRET_KEY=test
      - CONNECT_INTERNAL_KEY_CONVERTER=org.apache.kafka.connect.json.JsonConverter
      - CONNECT_INTERNAL_VALUE_CONVERTER=org.apache.kafka.connect.json.JsonConverter
    volumes:
      - ./localstack:/tmp/localstack
    networks:
      - network_test

volumes:
  test_postgresql_data:
networks:
  network_test:
    driver: bridge
0 Answers
Related