diff --git a/openssh/map.jinja b/openssh/map.jinja index 018e7e1..0363a76 100644 --- a/openssh/map.jinja +++ b/openssh/map.jinja @@ -8,69 +8,102 @@ {#- Where to lookup parameters source files #} {%- set map_sources_dir = tplroot ~ "/parameters" %} -{#- Load defaults first to allow per formula default map.jinja configuration #} -{%- set _defaults_filename = map_sources_dir ~ "/defaults.yaml" %} -{%- do salt["log.debug"]( - "map.jinja: initialise parameters from " - ~ _defaults_filename - ) %} -{%- import_yaml _defaults_filename as default_settings %} - {#- List of sources to lookup for parameters #} -{%- do salt["log.debug"]("map.jinja: lookup 'map_jinja' configuration sources") %} {#- Fallback to previously used grains plus minion `id` #} {%- set map_sources = [ - "osarch", - "os_family", - "os", - "osfinger", - "config_get_lookup", - "config_get", - "id", + "Y:G@osarch", + "Y:G@os_family", + "Y:G@os", + "Y:G@osfinger", + "C@" ~ tplroot ~ ":lookup", + "C@" ~ tplroot, + "Y:G@id", ] %} -{#- Configure map.jinja from defaults.yaml #} -{%- set map_sources = default_settings | traverse( - "values:map_jinja:sources", - map_sources, +{%- do salt["log.debug"]( + "map.jinja: built-in configuration sources:\n" + ~ {"values": { + "map_jinja": {"sources": map_sources} + } + } + | yaml(False) ) %} -{#- Lookup global sources #} -{%- set map_sources = salt["config.get"]("map_jinja:sources", map_sources) %} -{#- Lookup per formula sources #} -{%- set map_sources = salt["config.get"]( - tplroot ~ ":map_jinja:sources", - map_sources, +{#- Allow centralised map.jinja configuration #} +{%- set _global_map_filename = "parameters/map_jinja.yaml" %} +{%- do salt["log.debug"]( + "map.jinja: load global map.jinja values from " + ~ _global_map_filename ) %} +{%- load_yaml as global_map_settings %} +{%- include _global_map_filename ignore missing %} +{%- endload %} + +{%- if global_map_settings %} +{%- do salt["log.debug"]( + "map.jinja: configure sources from global map.jinja configuration " + ~ _global_map_filename + ~ ":\n" + ~ {"map_jinja": global_map_settings} + | yaml(False) + ) %} +{%- set map_sources = global_map_settings + | traverse( + "values:sources", + map_sources, + ) %} +{%- endif %} + +{#- Allow per formula map.jinja configuration #} +{%- set _map_filename = map_sources_dir ~ "/map_jinja.yaml" %} +{%- do salt["log.debug"]( + "map.jinja: load per formula map.jinja values from " + ~ _map_filename + ) %} +{%- load_yaml as map_settings %} +{%- include _map_filename ignore missing %} +{%- endload %} + +{%- if map_settings %} +{%- do salt["log.debug"]( + "map.jinja: configure sources from formula map.jinja configuration " + ~ _map_filename + ~ ":\n" + ~ {"map_jinja": map_settings} + | yaml(False) + ) %} +{%- set map_sources = map_settings + | traverse( + "values:sources", + map_sources, + ) %} +{%- endif %} {%- do salt["log.debug"]( - "map.jinja: load parameters with sources from " + "map.jinja: load parameters from sources:\n" ~ map_sources + | yaml(False) ) %} -{#- Lookup with `config.get` from configurable roots #} +{#- Load formula defaults values #} +{%- set _defaults_filename = map_sources_dir ~ "/defaults.yaml" %} {%- do salt["log.debug"]( - "map.jinja: initialise 'config.get' roots with 'tplroot' " - ~ tplroot + "map.jinja: load per formula default values from " + ~ _defaults_filename ) %} -{%- set config_get_roots = [tplroot] %} -{#- Configure `config.get` from defaults.yaml #} -{%- set config_get_roots = default_settings | traverse( - "values:map_jinja:config_get_roots", - config_get_roots - ) %} -{#- Lookup global `config.get` roots #} -{%- set config_get_roots = salt["config.get"]( - "map_jinja:config_get_roots", - config_get_roots - ) %} -{#- Lookup per formula `config.get` roots #} -{%- set config_get_roots = salt["config.get"]( - tplroot ~ ":map_jinja:config_get_roots", - config_get_roots, - ) %} -{%- do salt["log.debug"]( - "map.jinja: load parameters with 'config.get' from roots " - ~ config_get_roots +{%- load_yaml as default_settings %} +{%- include _defaults_filename ignore missing %} +{%- endload %} + +{%- if not default_settings %} +{%- set default_settings = {'values': {} } %} +{%- endif %} + +{#- Make sure to track `map.jinja` configuration with `_mapdata` #} +{%- do default_settings["values"].update( + { + "map_jinja": map_settings + | traverse("values", {}) + } ) %} {#- Work around assignment inside for loop #} @@ -106,63 +139,115 @@ ) %} {%- endif %} +{%- set query_map = { + "C": "config.get", + "G": "grains.get", + "I": "pillar.get", + } %} -{#- process each `map.jinja` source #} +{#- Process each `map.jinja` source #} +{#- each source has a type: #} +{#- - `Y` to load values from YAML files (the default when no type is set) #} +{#- - `C` to lookup values with `config.get` #} +{#- - `G` to lookup values with `grains.get` #} +{#- - `I` to lookup values with `pillar.get` #} +{#- The YAML type option can define query type to build the file name: #} +{#- - `C` for query with `config.get` (the default when to query type is set) #} +{#- - `G` for query with `grains.get` #} +{#- - `I` for query with `pillar.get` #} +{#- The `C`, `G` or `I` types can define the `SUB` option #} +{#- to merge values in the sub key `mapdata.` instead of directly in `mapdata` #} {%- for map_source in map_sources %} -{%- if map_source in ["config_get", "config_get_lookup"] %} -{%- for _config_root in config_get_roots %} -{%- set _config_key = { - "config_get": _config_root, - "config_get_lookup": _config_root ~ ":lookup", - }.get(map_source) %} +{%- set source_parts = map_source.split('@') %} +{%- if source_parts|length == 1 %} +{#- By default we load YAML files for config looked up by `config.get` #} +{%- set source_type = "Y" %} +{%- set query_type = "C" %} +{%- set source_key = map_source %} +{%- elif source_parts[0][0] == "Y" %} +{%- set source_type = "Y" %} +{%- set query_type = source_parts[0].split(':')[1] | default("C") %} +{%- set source_key = source_parts[1] %} +{%- elif source_parts[0][0] in query_map.keys() %} +{%- set source_type = source_parts[0].split(':') | first %} +{%- set query_type = source_type %} +{%- set is_sub_key = source_parts[0].split(':')[1] | default(False) == "SUB" %} +{%- set source_key = source_parts[1] %} +{%- endif %} -{%- do salt["log.debug"]( - "map.jinja: retrieve '" - ~ _config_key - ~ "' with 'config.get'" - ~ _config["merge_msg"] - ) %} -{%- set _config_get = salt["config.get"]( - _config_key, default={}, **_config["merge_opt"] - ) %} +{%- set query_method = query_map[query_type] %} -{#- `slsutil.merge` defaults to `smart` instead of `None` for `config.get` #} -{%- set _strategy = _config["merge_strategy"] | default("smart", boolean=True) %} -{%- do salt["log.debug"]( - "map.jinja: merge '" - ~ _config_key - ~ "' retrieved with 'config.get'" - ~ ", merge: strategy='" - ~ _strategy - ~ "', lists='" - ~ _config["merge_lists"] - ~ "'" - ) %} +{%- if source_type in query_map.keys() %} +{#- Lookup source `@key:to:query` #} +{%- if source_type == "C" %} +{%- set merge_opts = _config["merge_opt"] %} +{%- set merge_msg = _config["merge_msg"] %} +{%- else %} +{#- No merging strategy supported for `grains.get` and `pillar.get` #} +{%- set merge_opts = {} %} +{%- set merge_msg = "" %} +{%- endif %} -{#- Keep values under each root key when there are more than one #} -{%- if config_get_roots|length > 1 %} -{%- set _config_get = { _config_root: _config_get } %} -{%- endif %} -{%- do _config.update( - { - "stack": salt["slsutil.merge"]( - _config["stack"], - _config_get, - strategy=_strategy, - merge_lists=_config["merge_lists"], - ) - } - ) %} -{%- endfor %} +{%- do salt["log.debug"]( + "map.jinja: retrieve '" + ~ source_key + ~ "' with '" + ~ query_method + ~ "'" + ~ merge_msg + ) %} +{%- set _config_get = salt[query_method]( + source_key, + default={}, + **merge_opts + ) %} + +{#- `slsutil.merge` defaults to `smart` instead of `None` for `config.get` #} +{%- set _strategy = _config["merge_strategy"] | default("smart", boolean=True) %} +{%- do salt["log.debug"]( + "map.jinja: merge " + ~ "sub key " * is_sub_key + ~ "'" + ~ source_key + ~ "' retrieved with '" + ~ query_method + ~ "', merge: strategy='" + ~ _strategy + ~ "', lists='" + ~ _config["merge_lists"] + ~ "'" + ) %} + +{%- if is_sub_key %} +{#- Merge values with `mapdata.`, `` and `:lookup` are merged together #} +{%- set _config_get = { source_key.rstrip(':lookup'): _config_get } %} +{%- endif %} +{%- do _config.update( + { + "stack": salt["slsutil.merge"]( + _config["stack"], + _config_get, + strategy=_strategy, + merge_lists=_config["merge_lists"], + ) + } + ) %} {%- else %} -{#- Lookup the grain/pillar/... #} +{#- Load YAML file matching the grain/pillar/... #} {#- Fallback to use the source name as a direct filename #} -{%- set map_values = salt["config.get"](map_source, []) %} +{%- do salt["log.debug"]( + "map.jinja: lookup '" + ~ source_key + ~ "' with '" + ~ query_method + ~ "'" + ) %} +{%- set map_values = salt[query_method](source_key, []) %} -{#- Mangle `map_source` to use it as literal path #} +{#- Mangle `source_key` to use it as literal path #} {%- if map_values | length == 0 %} -{%- set map_source_parts = map_source.split("/") %} -{%- set map_source = map_source_parts[0:-1] | join("/") %} +{%- set map_source_parts = source_key.split("/") %} +{%- set source_key = map_source_parts[0:-1] | join("/") %} {%- set map_values = map_source_parts[-1].rstrip(".yaml") %} {%- endif %} @@ -171,14 +256,20 @@ {%- set map_values = [map_values] %} {%- endif %} +{#- `source_key` can be an empty string with literal path like `myconf.yaml` #} +{%- set yaml_dir = [ + map_sources_dir, + source_key + ] + | select + | join("/") %} + {%- for map_value in map_values %} {%- set yamlfile = [ - map_sources_dir, - map_source, - map_value ~ ".yaml", + yaml_dir, + map_value ~ ".yaml" ] - | join("/") - %} + | join("/") %} {%- do salt["log.debug"]("map.jinja: load parameters from file " ~ yamlfile) %} {%- load_yaml as loaded_values %} {%- include yamlfile ignore missing %} diff --git a/openssh/parameters/defaults.yaml b/openssh/parameters/defaults.yaml index 2fe08d4..b7c4436 100644 --- a/openssh/parameters/defaults.yaml +++ b/openssh/parameters/defaults.yaml @@ -2,12 +2,6 @@ # vim: ft=yaml --- values: - map_jinja: - config_get_roots: - - openssh - - sshd_config - - ssh_config - openssh: sshd_enable: true sshd_binary: /usr/sbin/sshd diff --git a/openssh/parameters/map_jinja.yaml b/openssh/parameters/map_jinja.yaml new file mode 100644 index 0000000..7d479b0 --- /dev/null +++ b/openssh/parameters/map_jinja.yaml @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# vim: ft=yaml +--- +values: + sources: + - "Y:G@osarch" + - "Y:G@os_family" + - "Y:G@os" + - "Y:G@osfinger" + + # Merge values from `config.get` under `mapdata.` to keep + # compatibility with user pillars. + # The `` and `:lookup` are merged together + - "C:SUB@openssh:lookup" + - "C:SUB@openssh" + - "C:SUB@sshd_config:lookup" + - "C:SUB@sshd_config" + - "C:SUB@ssh_config:lookup" + - "C:SUB@ssh_config" + + - "Y:G@id" diff --git a/test/integration/default/files/_mapdata/amazonlinux-1.yaml b/test/integration/default/files/_mapdata/amazonlinux-1.yaml index c4ed9c9..5a1098f 100644 --- a/test/integration/default/files/_mapdata/amazonlinux-1.yaml +++ b/test/integration/default/files/_mapdata/amazonlinux-1.yaml @@ -3,10 +3,18 @@ --- values: map_jinja: - config_get_roots: - - openssh - - sshd_config - - ssh_config + sources: + - Y:G@osarch + - Y:G@os_family + - Y:G@os + - Y:G@osfinger + - C:SUB@openssh:lookup + - C:SUB@openssh + - C:SUB@sshd_config:lookup + - C:SUB@sshd_config + - C:SUB@ssh_config:lookup + - C:SUB@ssh_config + - Y:G@id openssh: absent_dsa_keys: false absent_ecdsa_keys: false diff --git a/test/integration/default/files/_mapdata/amazonlinux-2.yaml b/test/integration/default/files/_mapdata/amazonlinux-2.yaml index 5311f2c..605165d 100644 --- a/test/integration/default/files/_mapdata/amazonlinux-2.yaml +++ b/test/integration/default/files/_mapdata/amazonlinux-2.yaml @@ -3,10 +3,18 @@ --- values: map_jinja: - config_get_roots: - - openssh - - sshd_config - - ssh_config + sources: + - Y:G@osarch + - Y:G@os_family + - Y:G@os + - Y:G@osfinger + - C:SUB@openssh:lookup + - C:SUB@openssh + - C:SUB@sshd_config:lookup + - C:SUB@sshd_config + - C:SUB@ssh_config:lookup + - C:SUB@ssh_config + - Y:G@id openssh: absent_dsa_keys: false absent_ecdsa_keys: false diff --git a/test/integration/default/files/_mapdata/arch-base-latest.yaml b/test/integration/default/files/_mapdata/arch-base-latest.yaml index 8f62a0e..a3fffb5 100644 --- a/test/integration/default/files/_mapdata/arch-base-latest.yaml +++ b/test/integration/default/files/_mapdata/arch-base-latest.yaml @@ -3,10 +3,18 @@ --- values: map_jinja: - config_get_roots: - - openssh - - sshd_config - - ssh_config + sources: + - Y:G@osarch + - Y:G@os_family + - Y:G@os + - Y:G@osfinger + - C:SUB@openssh:lookup + - C:SUB@openssh + - C:SUB@sshd_config:lookup + - C:SUB@sshd_config + - C:SUB@ssh_config:lookup + - C:SUB@ssh_config + - Y:G@id openssh: absent_dsa_keys: false absent_ecdsa_keys: false diff --git a/test/integration/default/files/_mapdata/centos-6.yaml b/test/integration/default/files/_mapdata/centos-6.yaml index 1f88c5f..26145db 100644 --- a/test/integration/default/files/_mapdata/centos-6.yaml +++ b/test/integration/default/files/_mapdata/centos-6.yaml @@ -3,10 +3,18 @@ --- values: map_jinja: - config_get_roots: - - openssh - - sshd_config - - ssh_config + sources: + - Y:G@osarch + - Y:G@os_family + - Y:G@os + - Y:G@osfinger + - C:SUB@openssh:lookup + - C:SUB@openssh + - C:SUB@sshd_config:lookup + - C:SUB@sshd_config + - C:SUB@ssh_config:lookup + - C:SUB@ssh_config + - Y:G@id openssh: absent_dsa_keys: false absent_ecdsa_keys: false diff --git a/test/integration/default/files/_mapdata/centos-7.yaml b/test/integration/default/files/_mapdata/centos-7.yaml index dffc0a7..63eac30 100644 --- a/test/integration/default/files/_mapdata/centos-7.yaml +++ b/test/integration/default/files/_mapdata/centos-7.yaml @@ -3,10 +3,18 @@ --- values: map_jinja: - config_get_roots: - - openssh - - sshd_config - - ssh_config + sources: + - Y:G@osarch + - Y:G@os_family + - Y:G@os + - Y:G@osfinger + - C:SUB@openssh:lookup + - C:SUB@openssh + - C:SUB@sshd_config:lookup + - C:SUB@sshd_config + - C:SUB@ssh_config:lookup + - C:SUB@ssh_config + - Y:G@id openssh: absent_dsa_keys: false absent_ecdsa_keys: false diff --git a/test/integration/default/files/_mapdata/centos-8.yaml b/test/integration/default/files/_mapdata/centos-8.yaml index d3ea759..991ad4b 100644 --- a/test/integration/default/files/_mapdata/centos-8.yaml +++ b/test/integration/default/files/_mapdata/centos-8.yaml @@ -3,10 +3,18 @@ --- values: map_jinja: - config_get_roots: - - openssh - - sshd_config - - ssh_config + sources: + - Y:G@osarch + - Y:G@os_family + - Y:G@os + - Y:G@osfinger + - C:SUB@openssh:lookup + - C:SUB@openssh + - C:SUB@sshd_config:lookup + - C:SUB@sshd_config + - C:SUB@ssh_config:lookup + - C:SUB@ssh_config + - Y:G@id openssh: absent_dsa_keys: false absent_ecdsa_keys: false diff --git a/test/integration/default/files/_mapdata/debian-10.yaml b/test/integration/default/files/_mapdata/debian-10.yaml index bc3bcc0..b41f767 100644 --- a/test/integration/default/files/_mapdata/debian-10.yaml +++ b/test/integration/default/files/_mapdata/debian-10.yaml @@ -3,10 +3,18 @@ --- values: map_jinja: - config_get_roots: - - openssh - - sshd_config - - ssh_config + sources: + - Y:G@osarch + - Y:G@os_family + - Y:G@os + - Y:G@osfinger + - C:SUB@openssh:lookup + - C:SUB@openssh + - C:SUB@sshd_config:lookup + - C:SUB@sshd_config + - C:SUB@ssh_config:lookup + - C:SUB@ssh_config + - Y:G@id openssh: absent_dsa_keys: false absent_ecdsa_keys: false diff --git a/test/integration/default/files/_mapdata/debian-9.yaml b/test/integration/default/files/_mapdata/debian-9.yaml index 185dae0..3f5ed73 100644 --- a/test/integration/default/files/_mapdata/debian-9.yaml +++ b/test/integration/default/files/_mapdata/debian-9.yaml @@ -3,10 +3,18 @@ --- values: map_jinja: - config_get_roots: - - openssh - - sshd_config - - ssh_config + sources: + - Y:G@osarch + - Y:G@os_family + - Y:G@os + - Y:G@osfinger + - C:SUB@openssh:lookup + - C:SUB@openssh + - C:SUB@sshd_config:lookup + - C:SUB@sshd_config + - C:SUB@ssh_config:lookup + - C:SUB@ssh_config + - Y:G@id openssh: absent_dsa_keys: false absent_ecdsa_keys: false diff --git a/test/integration/default/files/_mapdata/fedora-31.yaml b/test/integration/default/files/_mapdata/fedora-31.yaml index 53bd782..a8ee627 100644 --- a/test/integration/default/files/_mapdata/fedora-31.yaml +++ b/test/integration/default/files/_mapdata/fedora-31.yaml @@ -3,10 +3,18 @@ --- values: map_jinja: - config_get_roots: - - openssh - - sshd_config - - ssh_config + sources: + - Y:G@osarch + - Y:G@os_family + - Y:G@os + - Y:G@osfinger + - C:SUB@openssh:lookup + - C:SUB@openssh + - C:SUB@sshd_config:lookup + - C:SUB@sshd_config + - C:SUB@ssh_config:lookup + - C:SUB@ssh_config + - Y:G@id openssh: absent_dsa_keys: false absent_ecdsa_keys: false diff --git a/test/integration/default/files/_mapdata/fedora-32.yaml b/test/integration/default/files/_mapdata/fedora-32.yaml index eedcbd5..4088e15 100644 --- a/test/integration/default/files/_mapdata/fedora-32.yaml +++ b/test/integration/default/files/_mapdata/fedora-32.yaml @@ -3,10 +3,18 @@ --- values: map_jinja: - config_get_roots: - - openssh - - sshd_config - - ssh_config + sources: + - Y:G@osarch + - Y:G@os_family + - Y:G@os + - Y:G@osfinger + - C:SUB@openssh:lookup + - C:SUB@openssh + - C:SUB@sshd_config:lookup + - C:SUB@sshd_config + - C:SUB@ssh_config:lookup + - C:SUB@ssh_config + - Y:G@id openssh: absent_dsa_keys: false absent_ecdsa_keys: false diff --git a/test/integration/default/files/_mapdata/opensuse-15.yaml b/test/integration/default/files/_mapdata/opensuse-15.yaml index 5c7f1b1..f4a741c 100644 --- a/test/integration/default/files/_mapdata/opensuse-15.yaml +++ b/test/integration/default/files/_mapdata/opensuse-15.yaml @@ -3,10 +3,18 @@ --- values: map_jinja: - config_get_roots: - - openssh - - sshd_config - - ssh_config + sources: + - Y:G@osarch + - Y:G@os_family + - Y:G@os + - Y:G@osfinger + - C:SUB@openssh:lookup + - C:SUB@openssh + - C:SUB@sshd_config:lookup + - C:SUB@sshd_config + - C:SUB@ssh_config:lookup + - C:SUB@ssh_config + - Y:G@id openssh: absent_dsa_keys: false absent_ecdsa_keys: false diff --git a/test/integration/default/files/_mapdata/ubuntu-16.yaml b/test/integration/default/files/_mapdata/ubuntu-16.yaml index 268bc13..aff89d2 100644 --- a/test/integration/default/files/_mapdata/ubuntu-16.yaml +++ b/test/integration/default/files/_mapdata/ubuntu-16.yaml @@ -3,10 +3,18 @@ --- values: map_jinja: - config_get_roots: - - openssh - - sshd_config - - ssh_config + sources: + - Y:G@osarch + - Y:G@os_family + - Y:G@os + - Y:G@osfinger + - C:SUB@openssh:lookup + - C:SUB@openssh + - C:SUB@sshd_config:lookup + - C:SUB@sshd_config + - C:SUB@ssh_config:lookup + - C:SUB@ssh_config + - Y:G@id openssh: absent_dsa_keys: false absent_ecdsa_keys: false diff --git a/test/integration/default/files/_mapdata/ubuntu-18.yaml b/test/integration/default/files/_mapdata/ubuntu-18.yaml index 9330166..f592970 100644 --- a/test/integration/default/files/_mapdata/ubuntu-18.yaml +++ b/test/integration/default/files/_mapdata/ubuntu-18.yaml @@ -3,10 +3,18 @@ --- values: map_jinja: - config_get_roots: - - openssh - - sshd_config - - ssh_config + sources: + - Y:G@osarch + - Y:G@os_family + - Y:G@os + - Y:G@osfinger + - C:SUB@openssh:lookup + - C:SUB@openssh + - C:SUB@sshd_config:lookup + - C:SUB@sshd_config + - C:SUB@ssh_config:lookup + - C:SUB@ssh_config + - Y:G@id openssh: absent_dsa_keys: false absent_ecdsa_keys: false diff --git a/test/integration/default/files/_mapdata/ubuntu-20.yaml b/test/integration/default/files/_mapdata/ubuntu-20.yaml index 948b878..93a2a39 100644 --- a/test/integration/default/files/_mapdata/ubuntu-20.yaml +++ b/test/integration/default/files/_mapdata/ubuntu-20.yaml @@ -3,10 +3,18 @@ --- values: map_jinja: - config_get_roots: - - openssh - - sshd_config - - ssh_config + sources: + - Y:G@osarch + - Y:G@os_family + - Y:G@os + - Y:G@osfinger + - C:SUB@openssh:lookup + - C:SUB@openssh + - C:SUB@sshd_config:lookup + - C:SUB@sshd_config + - C:SUB@ssh_config:lookup + - C:SUB@ssh_config + - Y:G@id openssh: absent_dsa_keys: false absent_ecdsa_keys: false