Salt/openssh-formula
0
0
Fork 0
mirror of https://github.com/saltstack-formulas/openssh-formula.git synced 2026-07-23 20:23:29 +02:00
Code Issues Activity

refactor(map): load formula configuration with libmatchers

Browse Source
This commit is contained in:
Daniel Dehennin
2021-01-11 19:00:47 +01:00
parent 174bb68432
commit ff6b56c4a4
2 changed files with 262 additions and 205 deletions
Download Patch File Download Diff File Expand all files Collapse all files
openssh/libmapstack.jinja
+246 -73
View File
@@ -3,39 +3,104 @@
{#- Get the `tplroot` from `tpldir` #}
{%- set tplroot = tpldir.split("/")[0] %}
{%- from tplroot ~ "/libmatchers.jinja" import parse_matchers, query_map %}
{%- set _default_config_dirs = [
"parameters/",
tplroot ~ "/parameters"
] %}
{%- macro mapstack(
files,
matchers,
defaults=None,
dirs=_default_config_dirs,
log_prefix="libmapstack: "
) %}
{#-
Load YAML files in the order of `files` and merge successively the
values with `defaults` if the file exists.
Load configuration in the order of `matchers` and merge
successively the values with `defaults`.
The `matchers` are processed using `libmatchers.jinja` to select
the configuration sources from where the values are loaded.
Parameters:
- `files`: list of files to load
- `defaults`: default values to start the merging, they are
considered built-ins
- `matchers`: list of matchers in the form
`[<TYPE>[:<OPTION>[:<DELIMITER>]]@]<QUERY>`
- `defaults`: dictionary of default values to start the merging,
they are considered built-ins. It must conform to the same
layout as the YAML files: a mandatory `values` key and two
optional `strategy` and `merge_lists` keys.
- `dirs`: list of directory where to look-up the configuration
file matching the matchers, by default a global `salt://parameters/`
and a per formula `salt://<tplroot>/parameters`
- `log_prefix`: prefix used in the log outputs, by default it is
`libmapstack: `
Each YAML file must conform to the following layout:
Example: On a Debian system with `roles=["nginx/server", "telegraf"]`
{%- set settings = mapstack(
matchers=[
"Y:G@os_family",
"I@" ~ tplroot,
"Y:C@roles",
],
dirs=["defaults", tplroot ~ "/parameters"],
)
| load_yaml %}
This will merge the values:
- starting with the default empty dictionary `{}` (no
`defaults` parameter)
- from the YAML files
- `salt://defaults/os_family/Debian.yaml`
- `salt://{{ tplroot }}/parameters/os_family/Debian.yaml`
- from the pillar `salt["pillar.get"](tplroot)`
- from the `nginx/server` YAML files:
- `salt://defaults/roles/nginx/server.yaml`
- `salt://{{ tplroot }}/parameters/roles/nginx/server.yaml`
- from the `telegraf` YAML files:
- `salt://defaults/roles/telegraf.yaml`
- `salt://{{ tplroot }}/parameters/roles/telegraf.yaml`
Each YAML file and the `defaults` parameters must conform to the
following layout:
- a mandatory `values` key to store the configuration values
- two optional keys to configure the use of `salt.slsutil.merge`
- an optional `strategy` key to configure the merging strategy,
for example `strategy: 'recurse'`, the default is `smart`
- an optional `strategy` key to configure the merging
strategy, for example `strategy: 'recurse'`, the default is
`smart`
- an optional `merge_lists` key to configure if lists should be
merged or overridden for the `recurse` and `overwrite`
- an optional `merge_lists` key to configure if lists should
be merged or overridden for the `recurse` and `overwrite`
strategies, for example `merge_lists: 'true'`
#}
{%- set stack = defaults | default({"values": {} }, boolean=True) %}
{#- Build configuration file names based on matchers #}
{%- set matchers = parse_matchers(
matchers,
log_prefix=log_prefix
)
| load_yaml %}
{%- do salt["log.debug"](
log_prefix
~ "built-in configuration:\n"
@@ -43,74 +108,182 @@
| yaml(False)
) %}
{%- for yaml_filename in files %}
{%- do salt["log.debug"](
log_prefix
~ "load configuration values from "
~ yaml_filename
) %}
{%- load_yaml as yaml_values %}
{%- include yaml_filename ignore missing %}
{%- endload %}
{%- do salt["log.debug"](
log_prefix
~ "loaded configuration values from "
~ yaml_filename
~ ":\n"
~ yaml_values
| yaml(False)
) %}
{%- if yaml_values %}
{#- Per YAML file `salt["slsutil.merge"]` options or use defaults #}
{%- set _strategy = yaml_values
{%- for param_dir in dirs %}
{%- for matcher in matchers %}
{#- `slsutil.merge` options from #}
{#- 1. the `value` #}
{#- 2. the `defaults` #}
{#- 3. the built-in #}
{%- set _strategy = matcher.value
| traverse(
"strategy",
defaults
| traverse(
"strategy",
defaults
| traverse(
"strategy",
"smart"
)
) %}
{%- set _merge_lists = yaml_values
"smart"
)
) %}
{%- set _merge_lists = matcher.value
| traverse(
"merge_lists",
defaults
| traverse(
"merge_lists",
defaults
| traverse(
"merge_lists",
False
)
False
)
| to_bool %}
)
| to_bool %}
{#- Update only the `values`, the `salt["slsutil.merge"]` options are never updated #}
{%- do stack.update(
{
"values": salt["slsutil.merge"](
stack["values"],
yaml_values
| traverse("values", {}),
strategy=_strategy,
merge_lists=_merge_lists,
)
}
) %}
{%- do salt["log.debug"](
log_prefix
~ "merged configuration values from "
~ yaml_filename
~ ", merge: strategy='"
~ _strategy
~ "', merge_lists='"
~ _merge_lists
~ "':\n"
~ {"values": stack["values"]}
| yaml(False)
) %}
{%- endif %}
{%- if matcher.type in query_map.keys() %}
{#- No value is an empty list, must be a dict for `stack.update` #}
{%- set normalized_value = matcher.value | default({}, boolean=True) %}
{#- Merge in `mapdata.<query>` instead of directly in `mapdata` #}
{%- set is_sub_key = matcher.option | default(False) == "SUB" %}
{%- if is_sub_key %}
{#- Merge values with `mapdata.<key>`, `<key>` and `<key>:lookup` are merged together #}
{%- set value = { matcher.query | regex_replace(":lookup$", ""): normalized_value } %}
{%- else %}
{%- set value = normalized_value %}
{%- endif %}
{%- do salt["log.debug"](
log_prefix
~ "merge "
~ "sub key " * is_sub_key
~ "'"
~ matcher.query
~ "' retrieved with '"
~ matcher.query_method
~ "', merge: strategy='"
~ _strategy
~ "', lists='"
~ _merge_lists
~ "':\n"
~ value
| yaml(False)
) %}
{%- do stack.update(
{
"values": salt["slsutil.merge"](
stack["values"],
value,
strategy=_strategy,
merge_lists=_merge_lists,
)
}
) %}
{%- else %}
{#- Load YAML file matching the grain/pillar/... #}
{#- Fallback to use the source name as a direct filename #}
{%- if matcher.value | length == 0 %}
{#- Mangle `matcher.value` to use it as literal path #}
{%- set query_parts = matcher.query.split("/") %}
{%- set yaml_dirname = query_parts[0:-1] | join("/") %}
{%- set yaml_names = query_parts[-1] %}
{%- else %}
{%- set yaml_dirname = matcher.query %}
{%- set yaml_names = matcher.value %}
{%- endif %}
{#- Some configuration return list #}
{%- if yaml_names is string %}
{%- set yaml_names = [yaml_names] %}
{%- endif %}
{#- `yaml_dirname` can be an empty string with literal path like `myconf.yaml` #}
{%- set yaml_dir = [
param_dir,
yaml_dirname
]
| select
| join("/") %}
{%- for yaml_name in yaml_names %}
{#- Make sure to have a `.yaml` extension #}
{#- Use `.rpartition` to strip last `.yaml` in `dir.yaml/file.yaml` #}
{%- set yaml_filename = [
yaml_dir.rstrip("/"),
yaml_name.rpartition(".yaml")
| reject("equalto", ".yaml")
| join
~ ".yaml"
]
| select
| join("/") %}
{%- do salt["log.debug"](
log_prefix
~ "load configuration values from "
~ yaml_filename
) %}
{%- load_yaml as yaml_values %}
{%- include yaml_filename ignore missing %}
{%- endload %}
{%- if yaml_values %}
{%- do salt["log.debug"](
log_prefix
~ "loaded configuration values from "
~ yaml_name
~ ":\n"
~ yaml_values
| yaml(False)
) %}
{#- `slsutil.merge` options from #}
{#- 1. the `value` #}
{#- 2. the `defaults` #}
{#- 3. the built-in #}
{%- set _strategy = yaml_values
| traverse(
"strategy",
defaults
| traverse(
"strategy",
"smart"
)
) %}
{%- set _merge_lists = yaml_values
| traverse(
"merge_lists",
defaults
| traverse(
"merge_lists",
False
)
)
| to_bool %}
{%- do stack.update(
{
"values": salt["slsutil.merge"](
stack["values"],
yaml_values
| traverse("values", {}),
strategy=_strategy,
merge_lists=_merge_lists,
)
}
) %}
{%- do salt["log.debug"](
log_prefix
~ "merged configuration values from "
~ yaml_name
~ ", merge: strategy='"
~ _strategy
~ "', merge_lists='"
~ _merge_lists
~ "':\n"
~ {"values": stack["values"]}
| yaml(False)
) %}
{%- endif %}
{%- endfor %}
{%- endif %}
{%- endfor %}
{%- endfor %}
{%- do salt["log.debug"](
@@ -121,7 +294,7 @@
) %}
{#- Output stack as YAML, caller should use with something like #}
{#- `{%- set config = flexible_config(prefix="foo") | load_yaml %}` #}
{#- `{%- set config = mapstack(matchers=["foo"]) | load_yaml %}` #}
{{ stack | yaml }}
{%- endmacro %}