Helm not Importing file correctly

Viewed 58

I am creating an Helm Chart and I am having problems when it comes to importing files:

apiVersion: v1
kind: ConfigMap
metadata:
  name: vcl-template
  namespace: {{.Release.Namespace}}
data:
    {{- (.Files.Glob "config/varnish/default.vcl.tmpl").AsConfig | nindent 2 }}
    {{- (.Files.Glob "config/varnish/nginx.conf").AsConfig | nindent 2 }}

This imports the file config/varnish/nginx.conf just fine but the file config/varnish/default.vcl.tmpl is imported with \n instead of newlines, so the data on the ConfigMap gets all buggy:

apiVersion: v1
kind: ConfigMap
metadata:
  name: vcl-template
  namespace: default
data:
  default.vcl.tmpl: "vcl 4.0;\n\nimport std;\nimport directors;\n\n{{ range .Frontends
    }}\nbackend {{ .Name }} {\n    .host = \"{{ .Host }}\";\n    .port = \"{{ .Port
    }}\";\n}\n{{- end }}\n\n{{ range .Backends }}\nbackend be-{{ .Name }} {\n    .host
    = \"{{ .Host }}\";\n    .port = \"{{ .Port }}\";\n}\n{{- end }}\n\nacl purge {\n
    \   \"127.0.0.1\";\n    \"localhost\";\n    \"::1\";\n    {{- range .Frontends }}\n
    \   \"{{ .Host }}\";\n    {{- end }}\n    {{- range .Backends }}\n    \"{{ .Host
    }}\";\n    {{- end }}\n}\n\nsub vcl_init {\n    new cluster = directors.hash();\n\n
    \   {{ range .Frontends -}}\n    cluster.add_backend({{ .Name }}, 1);\n    {{ end
    }}\n\n    new lb = directors.round_robin();\n\n    {{ range .Backends -}}\n    lb.add_backend(be-{{
    .Name }});\n    {{ end }}\n}\n\nsub vcl_recv {\n\n    unset req.http.x-cache;\n
    \   set req.backend_hint = cluster.backend(req.url);\n    set req.http.x-shard =
    req.backend_hint;\n    if (req.http.x-shard != server.identity) {\n        return(pass);\n
    \   }\n    set req.backend_hint = lb.backend();\n\n    if (req.method == \"PURGE\")
    {\n        if (client.ip !~ purge) {\n            return (synth(405, \"Method not
    allowed\"));\n        }\n        # To use the X-Pool header for purging varnish
    during automated deployments, make sure the X-Pool header\n        # has been added
    to the response in your backend server config. This is used, for example, by the\n
    \       # capistrano-magento2 gem for purging old content from varnish during it's
    deploy routine.\n        if (!req.http.X-Magento-Tags-Pattern && !req.http.X-Pool)
    {\n            return (synth(400, \"X-Magento-Tags-Pattern or X-Pool header required\"));\n
    \       }\n        if (req.http.X-Magento-Tags-Pattern) {\n            ban(\"obj.http.X-Magento-Tags
    ~ \" + req.http.X-Magento-Tags-Pattern);\n        }\n        if (req.http.X-Pool)
    {\n            ban(\"obj.http.X-Pool ~ \" + req.http.X-Pool);\n        }\n        return
    (synth(200, \"Purged\"));\n    }\n\n    if (req.method != \"GET\" &&\n        req.method
    != \"HEAD\" &&\n        req.method != \"PUT\" &&\n        req.method != \"POST\"
    &&\n        req.method != \"TRACE\" &&\n        req.method != \"OPTIONS\" &&\n        req.method
    != \"DELETE\") {\n            /* Non-RFC2616 or CONNECT which is weird. */\n            return
    (pipe);\n    }\n\n    # We only deal with GET and HEAD by default\n    if (req.method
    != \"GET\" && req.method != \"HEAD\") {\n        return (pass);\n    }\n\n    #
    Bypass shopping cart, checkout and search requests\n    if (req.url ~ \"/checkout\"
    || req.url ~ \"/catalogsearch\") {\n        return (pass);\n    }\n\n    # Bypass
    admin\n    if (req.url ~ \"^/admin($|/.*)\") {\n        return (pass);\n    }\n\n
    \   # Bypass health check requests\n    if (req.url ~ \"/pub/health_check.php\")
    {\n        return (pass);\n    }\n\n    # Set initial grace period usage status\n
    \   set req.http.grace = \"none\";\n\n    # normalize url in case of leading HTTP
    scheme and domain\n    set req.url = regsub(req.url, \"^http[s]?://\", \"\");\n\n
    \   # collect all cookies\n    std.collect(req.http.Cookie);\n\n    # Compression
    filter. See https://www.varnish-cache.org/trac/wiki/FAQ/Compression\n    if (req.http.Accept-Encoding)
    {\n        if (req.url ~ \"\\.(jpg|jpeg|png|gif|gz|tgz|bz2|tbz|mp3|ogg|swf|flv)$\")
    {\n            # No point in compressing these\n            unset req.http.Accept-Encoding;\n
    \       } elsif (req.http.Accept-Encoding ~ \"gzip\") {\n            set req.http.Accept-Encoding
    = \"gzip\";\n        } elsif (req.http.Accept-Encoding ~ \"deflate\" && req.http.user-agent
    !~ \"MSIE\") {\n            set req.http.Accept-Encoding = \"deflate\";\n        }
    else {\n            # unknown algorithm\n            unset req.http.Accept-Encoding;\n
    \       }\n    }\n\n    # Remove all marketing get parameters to minimize the cache
    objects\n    if (req.url ~ \"(\\?|&)(gclid|cx|ie|cof|siteurl|zanpid|origin|fbclid|mc_[a-z]+|utm_[a-z]+|_bta_[a-z]+)=\")
    {\n        set req.url = regsuball(req.url, \"(gclid|cx|ie|cof|siteurl|zanpid|origin|fbclid|mc_[a-z]+|utm_[a-z]+|_bta_[a-z]+)=[-_A-z0-9+()%.]+&?\",
    \"\");\n        set req.url = regsub(req.url, \"[?|&]+$\", \"\");\n    }\n\n    #
    Static files caching\n    if (req.url ~ \"^/(pub/)?(media|static)/\") {\n        return
    (pass);\n    }\n\n    return (hash);\n}\n\nsub vcl_hash {\n    if (req.http.cookie
    ~ \"X-Magento-Vary=\") {\n        hash_data(regsub(req.http.cookie, \"^.*?X-Magento-Vary=([^;]+);*.*$\",
    \"\\1\"));\n    }\n\n    # For multi site configurations to not cache each other's
    content\n    if (req.http.host) {\n        hash_data(req.http.host);\n    } else
    {\n        hash_data(server.ip);\n    }\n\n    if (req.url ~ \"/graphql\") {\n        call
    process_graphql_headers;\n    }\n\n    # To make sure http users don't see ssl warning\n
    \   if (req.http.X-Forwarded-Proto) {\n        hash_data(req.http.X-Forwarded-Proto);\n
    \   }\n    \n}\n\nsub process_graphql_headers {\n    if (req.http.Store) {\n        hash_data(req.http.Store);\n
    \   }\n    if (req.http.Content-Currency) {\n        hash_data(req.http.Content-Currency);\n
    \   }\n}\n\nsub vcl_backend_response {\n\n    set beresp.grace = 3d;\n\n    if (beresp.http.content-type
    ~ \"text\") {\n        set beresp.do_esi = true;\n    }\n\n    if (bereq.url ~ \"\\.js$\"
    || beresp.http.content-type ~ \"text\") {\n        set beresp.do_gzip = true;\n
    \   }\n\n    if (beresp.http.X-Magento-Debug) {\n        set beresp.http.X-Magento-Cache-Control
    = beresp.http.Cache-Control;\n    }\n\n    # cache only successfully responses and
    404s\n    if (beresp.status != 200 && beresp.status != 404) {\n        set beresp.ttl
    = 0s;\n        set beresp.uncacheable = true;\n        return (deliver);\n    }
    elsif (beresp.http.Cache-Control ~ \"private\") {\n        set beresp.uncacheable
    = true;\n        set beresp.ttl = 86400s;\n        return (deliver);\n    }\n\n
    \   # validate if we need to cache it and prevent from setting cookie\n    if (beresp.ttl
    > 0s && (bereq.method == \"GET\" || bereq.method == \"HEAD\")) {\n        unset
    beresp.http.set-cookie;\n    }\n\n    # If page is not cacheable then bypass varnish
    for 2 minutes as Hit-For-Pass\n    if (beresp.ttl <= 0s ||\n        beresp.http.Surrogate-control
    ~ \"no-store\" ||\n        (!beresp.http.Surrogate-Control &&\n        beresp.http.Cache-Control
    ~ \"no-cache|no-store\") ||\n        beresp.http.Vary == \"*\") {\n        # Mark
    as Hit-For-Pass for the next 2 minutes\n        set beresp.ttl = 120s;\n        set
    beresp.uncacheable = true;\n    }\n\n    return (deliver);\n}\n\nsub vcl_deliver
    {\n    if (resp.http.X-Magento-Debug) {\n        if (resp.http.x-varnish ~ \" \")
    {\n            set resp.http.X-Magento-Cache-Debug = \"HIT\";\n            set resp.http.Grace
    = req.http.grace;\n        } else {\n            set resp.http.X-Magento-Cache-Debug
    = \"MISS\";\n        }\n    } else {\n        unset resp.http.Age;\n    }\n\n    #
    Not letting browser to cache non-static files.\n    if (resp.http.Cache-Control
    !~ \"private\" && req.url !~ \"^/(pub/)?(media|static)/\") {\n        set resp.http.Pragma
    = \"no-cache\";\n        set resp.http.Expires = \"-1\";\n        set resp.http.Cache-Control
    = \"no-store, no-cache, must-revalidate, max-age=0\";\n    }\n\n    unset resp.http.X-Magento-Debug;\n
    \   unset resp.http.X-Magento-Tags;\n    unset resp.http.X-Powered-By;\n    unset
    resp.http.Server;\n    unset resp.http.X-Varnish;\n    unset resp.http.Via;\n    unset
    resp.http.Link;\n}\n\nsub vcl_hit {\n    if (obj.ttl >= 0s) {\n        # Hit within
    TTL period\n        return (deliver);\n    }\n    if (std.healthy(req.backend_hint))
    {\n        if (obj.ttl + 300s > 0s) {\n            # Hit after TTL expiration, but
    within grace period\n            set req.http.grace = \"normal (healthy server)\";\n
    \           return (deliver);\n        } else {\n            # Hit after TTL and
    grace expiration\n            return (miss);\n        }\n    } else {\n        #
    server is not healthy, retrieve from cache\n        set req.http.grace = \"unlimited
    (unhealthy server)\";\n        return (deliver);\n    }\n}\n"
  nginx.conf: |
    worker_processes auto;
  
    events {
        worker_connections 1024;
    }
  
    pcre_jit on;
    error_log /var/log/nginx/error.log warn;
    include /etc/nginx/modules/*.conf;
  
    http {
        include /etc/nginx/mime.types;
        default_type application/octet-stream;
        server_tokens off;
        client_max_body_size 15m;
        keepalive_timeout 30;
        sendfile on;
        tcp_nodelay on;
        gzip_vary on;
        log_format main '$remote_addr - $remote_user [$time_local] "$request" '
        '$status $body_bytes_sent "$http_referer" '
        '"$http_user_agent" "$http_x_forwarded_for"';
        access_log /var/log/nginx/access.log main;
        include /etc/nginx/conf.d/*.conf;
    }

nginx.conf:

worker_processes auto;

events {
    worker_connections 1024;
}

pcre_jit on;
error_log /var/log/nginx/error.log warn;
include /etc/nginx/modules/*.conf;

http {
    include /etc/nginx/mime.types;
    default_type application/octet-stream;
    server_tokens off;
    client_max_body_size 15m;
    keepalive_timeout 30;
    sendfile on;
    tcp_nodelay on;
    gzip_vary on;
    log_format main '$remote_addr - $remote_user [$time_local] "$request" '
    '$status $body_bytes_sent "$http_referer" '
    '"$http_user_agent" "$http_x_forwarded_for"';
    access_log /var/log/nginx/access.log main;
    include /etc/nginx/conf.d/*.conf;
}

default.vcl.tmpl:

vcl 4.0;

import std;
import directors;

{{ range .Frontends }}
backend {{ .Name }} {
    .host = "{{ .Host }}";
    .port = "{{ .Port }}";
}
{{- end }}

{{ range .Backends }}
backend be-{{ .Name }} {
    .host = "{{ .Host }}";
    .port = "{{ .Port }}";
}
{{- end }}

acl purge {
    "127.0.0.1";
    "localhost";
    "::1";
    {{- range .Frontends }}
    "{{ .Host }}";
    {{- end }}
    {{- range .Backends }}
    "{{ .Host }}";
    {{- end }}
}

sub vcl_init {
    new cluster = directors.hash();

    {{ range .Frontends -}}
    cluster.add_backend({{ .Name }}, 1);
    {{ end }}

    new lb = directors.round_robin();

    {{ range .Backends -}}
    lb.add_backend(be-{{ .Name }});
    {{ end }}
}

sub vcl_recv {

    unset req.http.x-cache;
    set req.backend_hint = cluster.backend(req.url);
    set req.http.x-shard = req.backend_hint;
    if (req.http.x-shard != server.identity) {
        return(pass);
    }
    set req.backend_hint = lb.backend();

    if (req.method == "PURGE") {
        if (client.ip !~ purge) {
            return (synth(405, "Method not allowed"));
        }
        # To use the X-Pool header for purging varnish during automated deployments, make sure the X-Pool header
        # has been added to the response in your backend server config. This is used, for example, by the
        # capistrano-magento2 gem for purging old content from varnish during it's deploy routine.
        if (!req.http.X-Magento-Tags-Pattern && !req.http.X-Pool) {
            return (synth(400, "X-Magento-Tags-Pattern or X-Pool header required"));
        }
        if (req.http.X-Magento-Tags-Pattern) {
            ban("obj.http.X-Magento-Tags ~ " + req.http.X-Magento-Tags-Pattern);
        }
        if (req.http.X-Pool) {
            ban("obj.http.X-Pool ~ " + req.http.X-Pool);
        }
        return (synth(200, "Purged"));
    }

    if (req.method != "GET" &&
        req.method != "HEAD" &&
        req.method != "PUT" &&
        req.method != "POST" &&
        req.method != "TRACE" &&
        req.method != "OPTIONS" &&
        req.method != "DELETE") {
            /* Non-RFC2616 or CONNECT which is weird. */
            return (pipe);
    }

    # We only deal with GET and HEAD by default
    if (req.method != "GET" && req.method != "HEAD") {
        return (pass);
    }

    # Bypass shopping cart, checkout and search requests
    if (req.url ~ "/checkout" || req.url ~ "/catalogsearch") {
        return (pass);
    }

    # Bypass admin
    if (req.url ~ "^/admin($|/.*)") {
        return (pass);
    }

    # Bypass health check requests
    if (req.url ~ "/pub/health_check.php") {
        return (pass);
    }

    # Set initial grace period usage status
    set req.http.grace = "none";

    # normalize url in case of leading HTTP scheme and domain
    set req.url = regsub(req.url, "^http[s]?://", "");

    # collect all cookies
    std.collect(req.http.Cookie);

    # Compression filter. See https://www.varnish-cache.org/trac/wiki/FAQ/Compression
    if (req.http.Accept-Encoding) {
        if (req.url ~ "\.(jpg|jpeg|png|gif|gz|tgz|bz2|tbz|mp3|ogg|swf|flv)$") {
            # No point in compressing these
            unset req.http.Accept-Encoding;
        } elsif (req.http.Accept-Encoding ~ "gzip") {
            set req.http.Accept-Encoding = "gzip";
        } elsif (req.http.Accept-Encoding ~ "deflate" && req.http.user-agent !~ "MSIE") {
            set req.http.Accept-Encoding = "deflate";
        } else {
            # unknown algorithm
            unset req.http.Accept-Encoding;
        }
    }

    # Remove all marketing get parameters to minimize the cache objects
    if (req.url ~ "(\?|&)(gclid|cx|ie|cof|siteurl|zanpid|origin|fbclid|mc_[a-z]+|utm_[a-z]+|_bta_[a-z]+)=") {
        set req.url = regsuball(req.url, "(gclid|cx|ie|cof|siteurl|zanpid|origin|fbclid|mc_[a-z]+|utm_[a-z]+|_bta_[a-z]+)=[-_A-z0-9+()%.]+&?", "");
        set req.url = regsub(req.url, "[?|&]+$", "");
    }

    # Static files caching
    if (req.url ~ "^/(pub/)?(media|static)/") {
        return (pass);
    }

    return (hash);
}

sub vcl_hash {
    if (req.http.cookie ~ "X-Magento-Vary=") {
        hash_data(regsub(req.http.cookie, "^.*?X-Magento-Vary=([^;]+);*.*$", "\1"));
    }

    # For multi site configurations to not cache each other's content
    if (req.http.host) {
        hash_data(req.http.host);
    } else {
        hash_data(server.ip);
    }

    if (req.url ~ "/graphql") {
        call process_graphql_headers;
    }

    # To make sure http users don't see ssl warning
    if (req.http.X-Forwarded-Proto) {
        hash_data(req.http.X-Forwarded-Proto);
    }
    
}

sub process_graphql_headers {
    if (req.http.Store) {
        hash_data(req.http.Store);
    }
    if (req.http.Content-Currency) {
        hash_data(req.http.Content-Currency);
    }
}

sub vcl_backend_response {

    set beresp.grace = 3d;

    if (beresp.http.content-type ~ "text") {
        set beresp.do_esi = true;
    }

    if (bereq.url ~ "\.js$" || beresp.http.content-type ~ "text") {
        set beresp.do_gzip = true;
    }

    if (beresp.http.X-Magento-Debug) {
        set beresp.http.X-Magento-Cache-Control = beresp.http.Cache-Control;
    }

    # cache only successfully responses and 404s
    if (beresp.status != 200 && beresp.status != 404) {
        set beresp.ttl = 0s;
        set beresp.uncacheable = true;
        return (deliver);
    } elsif (beresp.http.Cache-Control ~ "private") {
        set beresp.uncacheable = true;
        set beresp.ttl = 86400s;
        return (deliver);
    }

    # validate if we need to cache it and prevent from setting cookie
    if (beresp.ttl > 0s && (bereq.method == "GET" || bereq.method == "HEAD")) {
        unset beresp.http.set-cookie;
    }

    # If page is not cacheable then bypass varnish for 2 minutes as Hit-For-Pass
    if (beresp.ttl <= 0s ||
        beresp.http.Surrogate-control ~ "no-store" ||
        (!beresp.http.Surrogate-Control &&
        beresp.http.Cache-Control ~ "no-cache|no-store") ||
        beresp.http.Vary == "*") {
        # Mark as Hit-For-Pass for the next 2 minutes
        set beresp.ttl = 120s;
        set beresp.uncacheable = true;
    }

    return (deliver);
}

sub vcl_deliver {
    if (resp.http.X-Magento-Debug) {
        if (resp.http.x-varnish ~ " ") {
            set resp.http.X-Magento-Cache-Debug = "HIT";
            set resp.http.Grace = req.http.grace;
        } else {
            set resp.http.X-Magento-Cache-Debug = "MISS";
        }
    } else {
        unset resp.http.Age;
    }

    # Not letting browser to cache non-static files.
    if (resp.http.Cache-Control !~ "private" && req.url !~ "^/(pub/)?(media|static)/") {
        set resp.http.Pragma = "no-cache";
        set resp.http.Expires = "-1";
        set resp.http.Cache-Control = "no-store, no-cache, must-revalidate, max-age=0";
    }

    unset resp.http.X-Magento-Debug;
    unset resp.http.X-Magento-Tags;
    unset resp.http.X-Powered-By;
    unset resp.http.Server;
    unset resp.http.X-Varnish;
    unset resp.http.Via;
    unset resp.http.Link;
}

sub vcl_hit {
    if (obj.ttl >= 0s) {
        # Hit within TTL period
        return (deliver);
    }
    if (std.healthy(req.backend_hint)) {
        if (obj.ttl + 300s > 0s) {
            # Hit after TTL expiration, but within grace period
            set req.http.grace = "normal (healthy server)";
            return (deliver);
        } else {
            # Hit after TTL and grace expiration
            return (miss);
        }
    } else {
        # server is not healthy, retrieve from cache
        set req.http.grace = "unlimited (unhealthy server)";
        return (deliver);
    }
}

How come that the second file is not imported correctly? Latest Helm version and latest Go version.

Anyone has any ideas? The encoding of both files on VSCode shows as UTF8.

1 Answers

They're actually equivalent from YAML's PoV, just not as pretty, but most important for your specific case it's because yaml cannot represent trailing whitespace without quoting it, which is what it did due to line 164 of your .tmpl file, as seen by the \n \n in:

    \   }\n    \n}\n\nsub process_graphql_headers {\n    if (req.http.Store) {\n        hash_data(req.http.Store);\n
$ sed -ne 164p default.vcl.tmpl | xxd
00000000: 2020 2020 0a                                 .

turning on "strip trailing whitespace" in your editor will help that, or for this specific case you can just fix line 164

Related