I have spent some time reading the code and docs of go-yaml, but I have not found any way to do this, except forking the project..
I want to extend the YAML unmarshaller so that it can accept a custom YAML tag (!include <file> in this case), which in turn would allow me to add support for including files. This is easily implemented with other YAML libraries, like in this answer.
Is there any way to accomplish this, using the public interface of the library (or another yaml library)?
Yes, this is possible (since v3). You can load the whole YAML file into a yaml.Node and then walk over the structure. The trick is that yaml.Node is an intermediate representation which you can only access if you define an unmarshaler.
For example:
package main
import (
"errors"
"fmt"
"io/ioutil"
"gopkg.in/yaml.v3"
)
// used for loading included files
type Fragment struct {
content *yaml.Node
}
func (f *Fragment) UnmarshalYAML(value *yaml.Node) error {
var err error
// process includes in fragments
f.content, err = resolveIncludes(value)
return err
}
type IncludeProcessor struct {
target interface{}
}
func (i *IncludeProcessor) UnmarshalYAML(value *yaml.Node) error {
resolved, err := resolveIncludes(value)
if err != nil {
return err
}
return resolved.Decode(i.target)
}
func resolveIncludes(node *yaml.Node) (*yaml.Node, error) {
if node.Tag == "!include" {
if node.Kind != yaml.ScalarNode {
return nil, errors.New("!include on a non-scalar node")
}
file, err := ioutil.ReadFile(node.Value)
if err != nil {
return nil, err
}
var f Fragment
err = yaml.Unmarshal(file, &f)
return f.content, err
}
if node.Kind == yaml.SequenceNode || node.Kind == yaml.MappingNode {
var err error
for i := range node.Content {
node.Content[i], err = resolveIncludes(node.Content[i])
if err != nil {
return nil, err
}
}
}
return node, nil
}
type MyStructure struct {
// this structure holds the values you want to load after processing
// includes, e.g.
Num int
}
func main() {
var s MyStructure
yaml.Unmarshal([]byte("!include foo.yaml"), &IncludeProcessor{&s})
fmt.Printf("Num: %v", s.Num)
}
Code prints Num: 42 when a file foo.yaml exists with the content num: 42.
Modified @flyx's original code a little to make it modular for adding custom resolvers.
package main
import (
"errors"
"fmt"
"io/ioutil"
"os"
"gopkg.in/yaml.v3"
)
var tagResolvers = make(map[string]func(*yaml.Node) (*yaml.Node, error))
type Fragment struct {
content *yaml.Node
}
func (f *Fragment) UnmarshalYAML(value *yaml.Node) error {
var err error
// process includes in fragments
f.content, err = resolveTags(value)
return err
}
type CustomTagProcessor struct {
target interface{}
}
func (i *CustomTagProcessor) UnmarshalYAML(value *yaml.Node) error {
resolved, err := resolveTags(value)
if err != nil {
return err
}
return resolved.Decode(i.target)
}
func resolveTags(node *yaml.Node) (*yaml.Node, error) {
for tag, fn := range tagResolvers {
if node.Tag == tag {
return fn(node)
}
}
if node.Kind == yaml.SequenceNode || node.Kind == yaml.MappingNode {
var err error
for i := range node.Content {
node.Content[i], err = resolveTags(node.Content[i])
if err != nil {
return nil, err
}
}
}
return node, nil
}
func resolveIncludes(node *yaml.Node) (*yaml.Node, error) {
if node.Kind != yaml.ScalarNode {
return nil, errors.New("!include on a non-scalar node")
}
file, err := ioutil.ReadFile(node.Value)
if err != nil {
return nil, err
}
var f Fragment
err = yaml.Unmarshal(file, &f)
return f.content, err
}
func resolveGetValueFromEnv(node *yaml.Node) (*yaml.Node, error) {
if node.Kind != yaml.ScalarNode {
return nil, errors.New("!getValueFromEnv on a non-scalar node")
}
value := os.Getenv(node.Value)
if value == "" {
return nil, fmt.Errorf("environment variable %v not set", node.Value)
}
var f Fragment
err := yaml.Unmarshal([]byte(value), &f)
return f.content, err
}
func AddResolvers(tag string, fn func(*yaml.Node) (*yaml.Node, error)) {
tagResolvers[tag] = fn
}
func main() {
// Register custom tag resolvers
AddResolvers("!include", resolveIncludes)
AddResolvers("!getValueFromEnv", resolveGetValueFromEnv)
type MyStructure struct {
// this structure holds the values you want to load after processing
// includes, e.g.
Num int
}
var s MyStructure
os.Setenv("FOO", `{"num": 42}`)
err := yaml.Unmarshal([]byte("!getValueFromEnv FOO"), &CustomTagProcessor{&s})
if err != nil {
panic("Error encountered during unmarshalling")
}
fmt.Printf("\nNum: %v", s.Num)
err = yaml.Unmarshal([]byte("!include foo.yaml"), &CustomTagProcessor{&s})
if err != nil {
panic("Error encountered during unmarshalling")
}
fmt.Printf("\nNum: %v", s.Num)
}