I'm super new to Go and I'm trying to convert a JSON schema which contains terraform specific attribute names to AWS API specific JSON schema in Go.
For example: The input is of type:
{
"egress": [
{
"cidr_blocks": [
"0.0.0.0/0"
],
"description": "Port 443",
"from_port": 443,
"protocol": "tcp",
"to_port": 443
}
],
"ingress": [
{
"cidr_blocks": [
"0.0.0.0/0"
],
"description": "Port 443",
"from_port": 443,
"protocol": "tcp",
"to_port": 443
}
],
"name": "my_sec_group",
"vpc_id": "${aws_vpc.my_vpc.id}"
}
And the desired output should be matching the AWS API object which would be something like this:
{
"SecurityGroups": [
{
"GroupName": "my_sec_group",
"IpPermissions": [
{
"FromPort": 443,
"IpProtocol": "tcp",
"IpRanges": [
{
"CidrIp": "0.0.0.0/0",
"Description": "Port 443"
}
],
"ToPort": 443
}
],
"IpPermissionsEgress": [
{
"FromPort": 443,
"IpProtocol": "tcp",
"IpRanges": [
{
"CidrIp": "0.0.0.0/0",
"Description": "Port 443"
}
],
"ToPort": 443
}
],
"VpcId": "${aws_vpc.my_vpc.id}"
}
]
}
What I have tried till now:
I. Create a generic JSON spec and use it to convert the input JSON to desired output
My generic JSON spec looks something like this:
{
"aws_security_group": [
{
"terraform_attribute": "name",
"attribute_type": "string",
"aws_attribute": "GroupName"
},
{
"terraform_attribute": "vpc_id",
"attribute_type": "string",
"aws_attribute": "VpcId"
},
{
"terraform_attribute": "description",
"attribute_type": "string",
"aws_attribute": "Description"
},
{
"terraform_attribute": "ingress",
"attribute_type": "list",
"aws_attribute": "IpPermissions",
"list_items": [
{
"terraform_attribute": "from_port",
"attribute_type": "string",
"aws_attribute": "FromPort"
},
{
"terraform_attribute": "to_port",
"attribute_type": "string",
"aws_attribute": "ToPort"
},
{
"terraform_attribute": "protocol",
"attribute_type": "string",
"aws_attribute": "IpProtocol"
},
{
"terraform_attribute": "cidr_blocks",
"attribute_type": "list",
"aws_attribute": "IpRanges",
"list_items": [
{
"terraform_attribute": "cidr_blocks.value",
"attribute_type": "string",
"aws_attribute": "CidrIp"
},
{
"terraform_attribute": "description",
"attribute_type": "string",
"aws_attribute": "Description"
}
]
}
]
},
{
"terraform_attribute": "egress",
"attribute_type": "list",
"aws_attribute": "IpPermissionsEgress",
"list_items": [
{
"terraform_attribute": "from_port",
"attribute_type": "string",
"aws_attribute": "FromPort"
},
{
"terraform_attribute": "to_port",
"attribute_type": "string",
"aws_attribute": "ToPort"
},
{
"terraform_attribute": "protocol",
"attribute_type": "string",
"aws_attribute": "IpProtocol"
},
{
"terraform_attribute": "cidr_blocks",
"attribute_type": "list",
"aws_attribute": "IpRanges",
"list_items": [
{
"terraform_attribute": "cidr_blocks.value",
"attribute_type": "string",
"aws_attribute": "CidrIp"
},
{
"terraform_attribute": "description",
"attribute_type": "string",
"aws_attribute": "Description"
}
]
}
]
}
]
}
Then I'm using this spec to convert the input JSON to desired output by recursively visiting each block within the input something like this:
for key, val := range configCopy {
for _, detail := range resourceDetails {
detail := detail.(map[string]interface{})
if detail["attribute_type"] == "string" && detail["terraform_attribute"] == key {
delete(configCopy, key)
configCopy[detail["aws_attribute"].(string)] = val
break
} else if detail["attribute_type"] == "list" && detail["terraform_attribute"] == key {
delete(configCopy, key)
configCopy[detail["aws_attribute"].(string)] =
buildListValue(val, detail["list_items"].([]interface{}))
}
}
}
func buildListValue(val interface{}, listItemVal []interface{}) []map[string]interface{} {
v := reflect.ValueOf(val)
var result []map[string]interface{}
fmt.Println("Val Kind:", v.Kind(), "Val String: ", v.String())
valCopy := make([]interface{}, v.Len())
if v.Kind() == reflect.Slice {
for i := 0; i < v.Len(); i++ {
valCopy[i] = v.Index(i).Interface()
}
}
for _, eachVal := range valCopy {
res := make(map[string]interface{})
f := reflect.ValueOf(eachVal)
if f.Kind() == reflect.Map {
for _, key := range f.MapKeys() {
fmt.Println("Key:", key.String(), " value:", f.MapIndex(key).Interface())
for _, listItem := range listItemVal {
listItem := listItem.(map[string]interface{})
if listItem["attribute_type"] == "string" && listItem["terraform_attribute"] == key.String() {
res[listItem["aws_attribute"].(string)] = f.MapIndex(key).Interface()
break
} else if listItem["attribute_type"] == "list" && listItem["terraform_attribute"] == key.String() {
res[listItem["aws_attribute"].(string)] =
buildListValue(f.MapIndex(key).Interface(), listItem["list_items"].([]interface{}))
break
}
}
}
} else if f.Kind() == reflect.String {
for _, listItem := range listItemVal {
listItem := listItem.(map[string]interface{})
if strings.HasSuffix(listItem["terraform_attribute"].(string), ".value") {
res[listItem["aws_attribute"].(string)] = f.String()
}
}
}
if len(res) > 0 {
result = append(result, res)
}
}
return result
}
Where configCopy is the input JSON. This approach works, but the problems associated with this approach are:
- It is a time consuming process as I need to manually create a spec for each resource type.
- Error prone as the transformation code is completely dependent on the manual spec being defined.
II. Tried using some open-source JSON transformers in Go like kazaam
Project kazaam takes a JSON string and a schema spec as input and produces the result using the given spec very similar to project Jolt.
I tried using a spec like this to convert the above input JSON:
[
{
"operation": "shift",
"spec": {
"SecurityGroups.Description": "description",
"SecurityGroups.GroupName": "name",
"SecurityGroups.VpcId": "vpc_id",
"SecurityGroups.IpPermissions.IpProtocol": "ingress[*].protocol",
"SecurityGroups.IpPermissions.FromPort": "ingress[*].from_port",
"SecurityGroups.IpPermissions.ToPort": "ingress[*].to_port",
"SecurityGroups.IpPermissions.IpRanges.CidrIp": "ingress[*].cidr_blocks[*]",
"SecurityGroups.IpPermissions.IpRanges.Description": "ingress[*].description",
"SecurityGroups.IpPermissionsEgress.IpProtocol": "egress[*].protocol",
"SecurityGroups.IpPermissionsEgress.FromPort": "egress[*].from_port",
"SecurityGroups.IpPermissionsEgress.ToPort": "egress[*].to_port",
"SecurityGroups.IpPermissionsEgress.IpRanges.CidrIp": "egress[*].cidr_blocks[*]",
"SecurityGroups.IpPermissionsEgress.IpRanges.Description": "egress[*].description"
}
}
]
But the result produced was something like the one shown below which is not what I was expecting:
{
"SecurityGroups": {
"IpPermissionsEgress": {
"FromPort": [
443
],
"ToPort": [
443
],
"IpRanges": {
"Description": [
"Port 443"
],
"CidrIp": [
[
"0.0.0.0/0"
]
]
},
"IpProtocol": [
"tcp"
]
},
"VpcId": "${aws_vpc.my_vpc.id}",
"IpPermissions": {
"FromPort": [
443
],
"ToPort": [
443
],
"IpRanges": {
"CidrIp": [
[
"0.0.0.0/0"
]
],
"Description": [
"Port 443"
]
},
"IpProtocol": [
"tcp"
]
},
"Description": null,
"GroupName": "my_sec_group"
}
}
Which is not as expected. I may be doing something wrong while constructing the kazaam spec, but I'm not really sure.
Is there any better/efficient way to address this use case? Any suggestions on this would be of great help.