Flatten a list objects that have multiple nested objects

Question

I am hoping someone can help me. I have a list of objects that has nested lists of objects at the same level. I want to flatten the data to include pieces from all objects.

My variable looks like this:

databases = [
  {
    name                = "DB1"
    comment             = "Test Terraform database"
    data_retention_days = 1
    environments = [
      {
        environment_prefixes = ["dev", "tst"]
        snowflake_account    = "ab1234"
      }
    ]
    schemas = [
      {
        schema_name         = "X"
        comment             = ""
        is_transient        = false
        is_managed          = false
        data_retention_days = 1
      },
      {
        schema_name         = "Y"
        comment             = ""
        is_transient        = false
        is_managed          = false
        data_retention_days = 1
    }]
  }
]

I would like the output to be this:

dbs = {
    "DB1|ab1234|X" = {
        database_name              = "DB1"
        comment                    = ""
        data_retention_days        = 1
        env_prefixes               = [
            "dev",
            "tst",
        ]
        snowflake_account          = "ab1234"
        schema_name                = "X"
        schema_comment             = ""
        schema_data_retention_days = 1
        is_transient               = false
        is_managed                 = false
    },
    "DB1|ab1234|Y" = {
        database_name              = "DB1"
        comment                    = ""
        data_retention_days        = 1
        env_prefixes               = [
            "dev",
            "tst",
        ]
        snowflake_account          = "ab1234"
        schema_name                = "Y"
        schema_comment             = ""
        schema_data_retention_days = 1
        is_transient               = false
        is_managed                 = false
    }
}

I have gotten pretty far but what is throwing me off is that the environments and schemas are at the same level I am not sure how to bring those together.

This flattens everything required down to the schemas level and works fine:

database_schemas = {
  for obj in flatten([
    for db in var.databases : [
      for schema in db.schemas : {
        database_name       = db.name
        schema_name         = schema.schema_name
        schema_comment      = schema.comment
        is_transient        = schema.is_transient
        is_managed          = schema.is_managed
        data_retention_days = schema.data_retention_days
      }
    ]
  ]) : "${obj.database_name}|${obj.schema_name}" => obj
}

This flattens everything down to the environments level and works fine:

database_environments = {
  for obj in flatten([
    for db in var.databases : [
      for env in db.environments : {
        database_name     = db.name
        env_prefixes      = env.environment_prefixes
        snowflake_account = env.snowflake_account
      }
    ]
  ]) : "${obj.database_name}|${obj.snowflake_account}" => obj
}

Since environments is at the same level as schemas, I am wondering how I can combine those? Essentially for each environment I want all schemas. Any thoughts or ideas are more than welcome! I appreciate the help in advance!

user_248823 · Answer

You can do it in one flatten operation with the help of setproduct:

locals {

  dbs = merge([
        for db in var.databases: {
           for idx, shema_env in setproduct(db.schemas, db.environments) :
             "${db.name}|${shema_env[1].snowflake_account}|${shema_env[0].schema_name}" => 
                 {
                  database_name       = db.name
                  schema_name         = shema_env[0].schema_name
                  schema_comment      = shema_env[0].comment
                  is_transient        = shema_env[0].is_transient
                  is_managed          = shema_env[0].is_managed
                  data_retention_days = shema_env[0].data_retention_days 
                  env_prefixes        = shema_env[1].environment_prefixes
                  snowflake_account   = shema_env[1].snowflake_account                  
                }  
        }
      ]...)  
}

which gives:

database_schemas = {
  "DB1|ab1234|X" = {
    "data_retention_days" = 1
    "database_name" = "DB1"
    "env_prefixes" = [
      "dev",
      "tst",
    ]
    "is_managed" = false
    "is_transient" = false
    "schema_comment" = ""
    "schema_name" = "X"
    "snowflake_account" = "ab1234"
  }
  "DB1|ab1234|Y" = {
    "data_retention_days" = 1
    "database_name" = "DB1"
    "env_prefixes" = [
      "dev",
      "tst",
    ]
    "is_managed" = false
    "is_transient" = false
    "schema_comment" = ""
    "schema_name" = "Y"
    "snowflake_account" = "ab1234"
  }
}

Flatten a list objects that have multiple nested objects

1 Answers