opensourcemirror
/
ansible
mirror of https://github.com/ansible/ansible.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Require Jinja2 3.0.0 (#75881) 

* Require Jinja2 3.0.0

ci_complete

* Fix sanity

* Remove Jinja min/max tests

* ansible-test changes

* ci_complete

* More cleanup

ci_complete

* Revert _count_newlines_from_end :( and other stuff

* Fix sanity

* It's using host_vars ...

* Unused import

* Remove overridden groupby filter

* environmentfilter -> pass_environment

* Explain preserve_trailing_newlines

* Add changelog

* ci_complete

* Deprecated ANSIBLE_JINJA2_NATIVE_WARNING

* native_helpers.py cleanup

* More cleanup in the find intgration test
This commit is contained in:
Martin Krizek 2021-10-20 19:22:29 +02:00 committed by GitHub
parent 835fe71828
commit 7621784b94
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
46 changed files with 98 additions and 523 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
changelogs/fragments/75881-jinja2-3-min.yml Normal file
View File

@ -0,0 +1,3 @@
major_changes:
- Jinja2 Controller Requirement - Jinja2 3.0.0 or newer is required for the control node (the machine that runs Ansible)
(https://github.com/ansible/ansible/pull/75881)

11
lib/ansible/__init__.py
View File

@ -22,17 +22,6 @@ __metaclass__ = type
# make vendored top-level modules accessible EARLY
import ansible._vendor
# patch Jinja2 >= 3.0 for backwards compatibility
try:
import sys as _sys
from jinja2.filters import pass_context as _passctx, pass_environment as _passenv, pass_eval_context as _passevalctx
_mod = _sys.modules['jinja2.filters']
_mod.contextfilter = _passctx
_mod.environmentfilter = _passenv
_mod.evalcontextfilter = _passevalctx
except ImportError:
_sys = None
# Note: Do not add any code to this file. The ansible module may be
# a namespace package when using Ansible-2.1+ Anything in this file may not be
# available if one of the other packages in the namespace is loaded first.

24
lib/ansible/config/base.yml
View File

@ -791,7 +791,7 @@ DEFAULT_JINJA2_EXTENSIONS:
DEFAULT_JINJA2_NATIVE:
name: Use Jinja2's NativeEnvironment for templating
default: False
description: This option preserves variable types during template operations. This requires Jinja2 >= 2.10.
description: This option preserves variable types during template operations.
env: [{name: ANSIBLE_JINJA2_NATIVE}]
ini:
- {key: jinja2_native, section: defaults}
@ -1656,6 +1656,19 @@ INVENTORY_UNPARSED_IS_FAILED:
ini:
- {key: unparsed_is_failed, section: inventory}
type: bool
JINJA2_NATIVE_WARNING:
name: Running older than required Jinja version for jinja2_native warning
default: True
description: Toggle to control showing warnings related to running a Jinja version
older than required for jinja2_native
env:
- name: ANSIBLE_JINJA2_NATIVE_WARNING
deprecated:
why: This option is no longer used in the Ansible Core code base.
version: "2.17"
ini:
- {key: jinja2_native_warning, section: defaults}
type: boolean
MAX_FILE_SIZE_FOR_DIFF:
name: Diff maximum file size
default: 104448
@ -1664,15 +1677,6 @@ MAX_FILE_SIZE_FOR_DIFF:
ini:
- {key: max_diff_size, section: defaults}
type: int
JINJA2_NATIVE_WARNING:
name: Running older than required Jinja version for jinja2_native warning
default: True
description: Toggle to control showing warnings related to running a Jinja version
older than required for jinja2_native
env: [{name: ANSIBLE_JINJA2_NATIVE_WARNING}]
ini:
- {key: jinja2_native_warning, section: defaults}
type: boolean
NETWORK_GROUP_MODULES:
name: Network module families
default: [eos, nxos, ios, iosxr, junos, enos, ce, vyos, sros, dellos9, dellos10, dellos6, asa, aruba, aireos, bigip, ironware, onyx, netconf, exos, voss, slxos]

12
lib/ansible/plugins/action/template.py
View File

@ -65,18 +65,6 @@ class ActionModule(ActionBase):
comment_end_string = self._task.args.get('comment_end_string', None)
output_encoding = self._task.args.get('output_encoding', 'utf-8') or 'utf-8'
# Option `lstrip_blocks' was added in Jinja2 version 2.7.
if lstrip_blocks:
try:
import jinja2.defaults
except ImportError:
raise AnsibleError('Unable to import Jinja2 defaults for determining Jinja2 features.')
try:
jinja2.defaults.LSTRIP_BLOCKS
except AttributeError:
raise AnsibleError("Option `lstrip_blocks' is only available in Jinja2 versions >=2.7")
wrong_sequences = ["\\n", "\\r", "\\r\\n"]
allowed_sequences = ["\n", "\r", "\r\n"]

1
lib/ansible/plugins/doc_fragments/template_common.py
View File

@ -90,7 +90,6 @@ options:
description:
- Determine when leading spaces and tabs should be stripped.
- When set to C(yes) leading spaces and tabs are stripped from the start of a line to a block.
- This functionality requires Jinja 2.7 or newer.
type: bool
default: no
version_added: '2.6'

31
lib/ansible/plugins/filter/core.py
View File

@ -21,7 +21,7 @@ import datetime
from functools import partial
from random import Random, SystemRandom, shuffle
from jinja2.filters import environmentfilter, do_groupby as _do_groupby
from jinja2.filters import pass_environment
from ansible.errors import AnsibleError, AnsibleFilterError, AnsibleFilterTypeError
from ansible.module_utils.six import string_types, integer_types, reraise, text_type
@ -32,7 +32,7 @@ from ansible.module_utils.common._collections_compat import Mapping
from ansible.module_utils.common.yaml import yaml_load, yaml_load_all
from ansible.parsing.ajson import AnsibleJSONEncoder
from ansible.parsing.yaml.dumper import AnsibleDumper
from ansible.template import AnsibleUndefined, recursive_check_defined
from ansible.template import recursive_check_defined
from ansible.utils.display import Display
from ansible.utils.encrypt import passlib_or_crypt
from ansible.utils.hashing import md5s, checksum_s
@ -217,7 +217,7 @@ def from_yaml_all(data):
return data
@environmentfilter
@pass_environment
def rand(environment, end, start=None, step=None, seed=None):
if seed is None:
r = SystemRandom()
@ -421,7 +421,7 @@ def comment(text, style='plain', **kw):
str_end)
@environmentfilter
@pass_environment
def extract(environment, item, container, morekeys=None):
if morekeys is None:
keys = [item]
@ -437,26 +437,6 @@ def extract(environment, item, container, morekeys=None):
return value
@environmentfilter
def do_groupby(environment, value, attribute):
"""Overridden groupby filter for jinja2, to address an issue with
jinja2>=2.9.0,<2.9.5 where a namedtuple was returned which
has repr that prevents ansible.template.safe_eval.safe_eval from being
able to parse and eval the data.
jinja2<2.9.0,>=2.9.5 is not affected, as <2.9.0 uses a tuple, and
>=2.9.5 uses a standard tuple repr on the namedtuple.
The adaptation here, is to run the jinja2 `do_groupby` function, and
cast all of the namedtuples to a regular tuple.
See https://github.com/ansible/ansible/issues/20098
We may be able to remove this in the future.
"""
return [tuple(t) for t in _do_groupby(environment, value, attribute)]
def b64encode(string, encoding='utf-8'):
return to_text(base64.b64encode(to_bytes(string, encoding=encoding, errors='surrogate_or_strict')))
@ -571,9 +551,6 @@ class FilterModule(object):
def filters(self):
return {
# jinja2 overrides
'groupby': do_groupby,
# base 64
'b64decode': b64decode,
'b64encode': b64encode,

45
lib/ansible/plugins/filter/mathstuff.py
View File

@ -26,7 +26,7 @@ __metaclass__ = type
import itertools
import math
from jinja2.filters import environmentfilter
from jinja2.filters import pass_environment
from ansible.errors import AnsibleFilterError, AnsibleFilterTypeError
from ansible.module_utils.common.text import formatters
@ -42,16 +42,11 @@ try:
except ImportError:
HAS_UNIQUE = False
try:
from jinja2.filters import do_max, do_min
HAS_MIN_MAX = True
except ImportError:
HAS_MIN_MAX = False
display = Display()
@environmentfilter
@pass_environment
# Use case_sensitive=None as a sentinel value, so we raise an error only when
# explicitly set and cannot be handle (by Jinja2 w/o 'unique' or fallback version)
def unique(environment, a, case_sensitive=None, attribute=None):
@ -88,7 +83,7 @@ def unique(environment, a, case_sensitive=None, attribute=None):
return c
@environmentfilter
@pass_environment
def intersect(environment, a, b):
if isinstance(a, Hashable) and isinstance(b, Hashable):
c = set(a) & set(b)
@ -97,7 +92,7 @@ def intersect(environment, a, b):
return c
@environmentfilter
@pass_environment
def difference(environment, a, b):
if isinstance(a, Hashable) and isinstance(b, Hashable):
c = set(a) - set(b)
@ -106,7 +101,7 @@ def difference(environment, a, b):
return c
@environmentfilter
@pass_environment
def symmetric_difference(environment, a, b):
if isinstance(a, Hashable) and isinstance(b, Hashable):
c = set(a) ^ set(b)
@ -116,7 +111,7 @@ def symmetric_difference(environment, a, b):
return c
@environmentfilter
@pass_environment
def union(environment, a, b):
if isinstance(a, Hashable) and isinstance(b, Hashable):
c = set(a) | set(b)
@ -125,30 +120,6 @@ def union(environment, a, b):
return c
@environmentfilter
def min(environment, a, **kwargs):
if HAS_MIN_MAX:
return do_min(environment, a, **kwargs)
else:
if kwargs:
raise AnsibleFilterError("Ansible's min filter does not support any keyword arguments. "
"You need Jinja2 2.10 or later that provides their version of the filter.")
_min = __builtins__.get('min')
return _min(a)
@environmentfilter
def max(environment, a, **kwargs):
if HAS_MIN_MAX:
return do_max(environment, a, **kwargs)
else:
if kwargs:
raise AnsibleFilterError("Ansible's max filter does not support any keyword arguments. "
"You need Jinja2 2.10 or later that provides their version of the filter.")
_max = __builtins__.get('max')
return _max(a)
def logarithm(x, base=math.e):
try:
if base == 10:
@ -251,10 +222,6 @@ class FilterModule(object):
def filters(self):
filters = {
# general math
'min': min,
'max': max,
# exponents and logarithms
'log': logarithm,
'pow': power,

57
lib/ansible/plugins/filter/urls.py
View File

@ -6,64 +6,15 @@
from __future__ import absolute_import, division, print_function
__metaclass__ = type
from ansible.module_utils.six import PY3, iteritems, string_types
from ansible.module_utils.six.moves.urllib.parse import quote, quote_plus, unquote_plus
from ansible.module_utils._text import to_bytes, to_text
from functools import partial
try:
from jinja2.filters import do_urlencode
HAS_URLENCODE = True
except ImportError:
HAS_URLENCODE = False
def unicode_urldecode(string):
if PY3:
return unquote_plus(string)
return to_text(unquote_plus(to_bytes(string)))
def do_urldecode(string):
return unicode_urldecode(string)
# NOTE: We implement urlencode when Jinja2 is older than v2.7
def unicode_urlencode(string, for_qs=False):
safe = b'' if for_qs else b'/'
if for_qs:
quote_func = quote_plus
else:
quote_func = quote
if PY3:
return quote_func(string, safe)
return to_text(quote_func(to_bytes(string), safe))
def do_urlencode(value):
itemiter = None
if isinstance(value, dict):
itemiter = iteritems(value)
elif not isinstance(value, string_types):
try:
itemiter = iter(value)
except TypeError:
pass
if itemiter is None:
return unicode_urlencode(value)
return u'&'.join(unicode_urlencode(k) + '=' +
unicode_urlencode(v, for_qs=True)
for k, v in itemiter)
from urllib.parse import unquote_plus
class FilterModule(object):
''' Ansible core jinja2 filters '''
def filters(self):
filters = {
'urldecode': do_urldecode,
return {
'urldecode': partial(unquote_plus),
}
if not HAS_URLENCODE:
filters['urlencode'] = do_urlencode
return filters

10
lib/ansible/plugins/lookup/template.py
View File

@ -79,13 +79,15 @@ _raw:
from copy import deepcopy
import os
import ansible.constants as C
from ansible.errors import AnsibleError
from ansible.plugins.lookup import LookupBase
from ansible.module_utils._text import to_bytes, to_text
from ansible.template import generate_ansible_template_vars, AnsibleEnvironment, USE_JINJA2_NATIVE
from ansible.template import generate_ansible_template_vars, AnsibleEnvironment
from ansible.utils.display import Display
if USE_JINJA2_NATIVE:
if C.DEFAULT_JINJA2_NATIVE:
from ansible.utils.native_jinja import NativeJinjaText
@ -109,7 +111,7 @@ class LookupModule(LookupBase):
comment_start_string = self.get_option('comment_start_string')
comment_end_string = self.get_option('comment_end_string')
if USE_JINJA2_NATIVE and not jinja2_native:
if C.DEFAULT_JINJA2_NATIVE and not jinja2_native:
templar = self._templar.copy_with_new_env(environment_class=AnsibleEnvironment)
else:
templar = self._templar
@ -152,7 +154,7 @@ class LookupModule(LookupBase):
res = templar.template(template_data, preserve_trailing_newlines=True,
convert_data=convert_data_p, escape_backslashes=False)
if USE_JINJA2_NATIVE and not jinja2_native:
if C.DEFAULT_JINJA2_NATIVE and not jinja2_native:
# jinja2_native is true globally but off for the lookup, we need this text
# not to be processed by literal_eval anywhere in Ansible
res = NativeJinjaText(res)

48
lib/ansible/template/__init__.py
View File

@ -28,7 +28,6 @@ import re
import time
from contextlib import contextmanager
from ansible.module_utils.compat.version import LooseVersion
from numbers import Number
from traceback import format_exc
@ -80,26 +79,14 @@ NON_TEMPLATED_TYPES = (bool, Number)
JINJA2_OVERRIDE = '#jinja2:'
from jinja2 import __version__ as j2_version
from jinja2 import Environment
from jinja2.utils import concat as j2_concat
USE_JINJA2_NATIVE = False
if C.DEFAULT_JINJA2_NATIVE:
try:
from jinja2.nativetypes import NativeEnvironment
from ansible.template.native_helpers import ansible_native_concat
from ansible.utils.native_jinja import NativeJinjaText
USE_JINJA2_NATIVE = True
except ImportError:
from jinja2 import Environment
from jinja2.utils import concat as j2_concat
if C.JINJA2_NATIVE_WARNING:
display.warning(
'jinja2_native requires Jinja 2.10 and above. '
'Version detected: %s. Falling back to default.' % j2_version
)
from jinja2.nativetypes import NativeEnvironment
from ansible.template.native_helpers import ansible_native_concat
from ansible.utils.native_jinja import NativeJinjaText
JINJA2_BEGIN_TOKENS = frozenset(('variable_begin', 'block_begin', 'comment_begin', 'raw_begin'))
@ -427,11 +414,10 @@ class AnsibleContext(Context):
Also see ``AnsibleJ2Template``and
https://github.com/pallets/jinja/commit/d67f0fd4cc2a4af08f51f4466150d49da7798729
"""
if LooseVersion(j2_version) >= LooseVersion('2.9'):
if not self.vars:
return self.parent
if not self.parent:
return self.vars
if not self.vars:
return self.parent
if not self.parent:
return self.vars
if isinstance(self.parent, AnsibleJ2Vars):
return self.parent.add_locals(self.vars)
@ -639,7 +625,7 @@ class AnsibleEnvironment(Environment):
self.tests = JinjaPluginIntercept(self.tests, test_loader, jinja2_native=False)
if USE_JINJA2_NATIVE:
if C.DEFAULT_JINJA2_NATIVE:
class AnsibleNativeEnvironment(NativeEnvironment):
'''
Our custom environment, which simply allows us to override the class-level
@ -675,7 +661,7 @@ class Templar:
self._fail_on_undefined_errors = C.DEFAULT_UNDEFINED_VAR_BEHAVIOR
environment_class = AnsibleNativeEnvironment if USE_JINJA2_NATIVE else AnsibleEnvironment
environment_class = AnsibleNativeEnvironment if C.DEFAULT_JINJA2_NATIVE else AnsibleEnvironment
self.environment = environment_class(
trim_blocks=True,
@ -737,7 +723,7 @@ class Templar:
if value is not None:
setattr(obj, key, value)
except AttributeError:
# Ignore invalid attrs, lstrip_blocks was added in jinja2==2.7
# Ignore invalid attrs
pass
return new_templar
@ -797,7 +783,7 @@ class Templar:
if value is not None:
setattr(obj, key, value)
except AttributeError:
# Ignore invalid attrs, lstrip_blocks was added in jinja2==2.7
# Ignore invalid attrs
pass
yield
@ -1151,13 +1137,11 @@ class Templar:
# calculate the difference in newlines and append them
# to the resulting output for parity
#
# jinja2 added a keep_trailing_newline option in 2.7 when
# creating an Environment. That would let us make this code
# better (remove a single newline if
# preserve_trailing_newlines is False). Once we can depend on
# that version being present, modify our code to set that when
# initializing self.environment and remove a single trailing
# newline here if preserve_newlines is False.
# Using Environment's keep_trailing_newline instead would
# result in change in behavior when trailing newlines
# would be kept also for included templates, for example:
# "Hello {% include 'world.txt' %}!" would render as
# "Hello world\n!\n" instead of "Hello world!\n".
res_newlines = _count_newlines_from_end(res)
if data_newlines > res_newlines:
res += self.environment.newline_sequence * (data_newlines - res_newlines)

11
lib/ansible/template/native_helpers.py
View File

@ -8,14 +8,12 @@ __metaclass__ = type
from ast import literal_eval
from itertools import islice, chain
import types
from jinja2.runtime import StrictUndefined
from ansible.module_utils._text import to_text
from ansible.module_utils.common.collections import is_sequence, Mapping
from ansible.module_utils.common.text.converters import container_to_text
from ansible.module_utils.six import text_type, string_types
from ansible.module_utils.six import string_types
from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode
from ansible.utils.native_jinja import NativeJinjaText
@ -78,12 +76,9 @@ def ansible_native_concat(nodes):
if not isinstance(out, string_types):
return out
else:
if isinstance(nodes, types.GeneratorType):
nodes = chain(head, nodes)
out = u''.join([to_text(_fail_on_undefined(v)) for v in nodes])
out = ''.join([to_text(_fail_on_undefined(v)) for v in chain(head, nodes)])
try:
out = literal_eval(out)
return out
return literal_eval(out)
except (ValueError, SyntaxError, MemoryError):
return out

1
lib/ansible/template/vars.py
View File

@ -113,7 +113,6 @@ class AnsibleJ2Vars(Mapping):
if locals is None:
return self
# FIXME run this only on jinja2>=2.9?
# prior to version 2.9, locals contained all of the vars and not just the current
# local vars so this was not necessary for locals to propagate down to nested includes
new_locals = self._locals.copy()

2
requirements.txt
View File

@ -3,7 +3,7 @@
# packages. Thus, this should be the loosest set possible (only required
# packages, not optional ones, and with the widest range of versions that could
# be suitable)
jinja2
jinja2 >= 3.0.0
PyYAML
cryptography
packaging

6
test/integration/targets/ansible-vault/single_vault_as_string.yml
View File

@ -81,8 +81,6 @@
- assert:
that:
- vaulted_value|map('upper')|list == ['F', 'O', 'O', ' ', 'B', 'A', 'R']
when: lookup('pipe', ansible_python.executable ~ ' -c "import jinja2; print(jinja2.__version__)"') is version('2.7', '>=')
- assert:
that:
@ -90,23 +88,19 @@
- vaulted_value|select('equalto', 'o')|list == ['o', 'o']
- vaulted_value|title == 'Foo Bar'
- vaulted_value is equalto('foo bar')
when: lookup('pipe', ansible_python.executable ~ ' -c "import jinja2; print(jinja2.__version__)"') is version('2.8', '>=')
- assert:
that:
- vaulted_value|string|tojson == '"foo bar"'
- vaulted_value|truncate(4) == 'foo bar'
when: lookup('pipe', ansible_python.executable ~ ' -c "import jinja2; print(jinja2.__version__)"') is version('2.9', '>=')
- assert:
that:
- vaulted_value|wordwrap(4) == 'foo\nbar'
when: lookup('pipe', ansible_python.executable ~ ' -c "import jinja2; print(jinja2.__version__)"') is version('2.11', '>=')
- assert:
that:
- vaulted_value|wordcount == 2
when: lookup('pipe', ansible_python.executable ~ ' -c "import jinja2; print(jinja2.__version__)"') is version('2.11.2', '>=')
- ping:
data: !vault |

3
test/integration/targets/collections/test_bypass_host_loop.yml
View File

@ -5,9 +5,6 @@
collections:
- testns.testcoll
tasks:
- meta: end_host
when: lookup('pipe', ansible_playbook_python ~ ' -c "import jinja2; print(jinja2.__version__)"') is version('2.7', '<')
- bypass_host_loop:
register: bypass

2
test/integration/targets/filter_core/tasks/main.yml
View File

@ -79,8 +79,6 @@
- "31 == ['x','y']|map('extract',{'x':42,'y':31})|list|last"
- "'local' == ['localhost']|map('extract',hostvars,'ansible_connection')|list|first"
- "'local' == ['localhost']|map('extract',hostvars,['ansible_connection'])|list|first"
# map was added to jinja2 in version 2.7
when: lookup('pipe', ansible_python.executable ~ ' -c "import jinja2; print(jinja2.__version__)"') is version('2.7', '>=')
- name: Test extract filter with defaults
vars:

10
test/integration/targets/filter_mathstuff/runme.sh
View File

@ -5,13 +5,3 @@ set -eux
export ANSIBLE_ROLES_PATH=../
ansible-playbook runme.yml "$@"
source virtualenv.sh
# Install Jinja < 2.10 since we want to test the fallback to Ansible's custom
# unique filter. Jinja < 2.10 does not have do_unique so we will trigger the
# fallback.
pip install 'jinja2 < 2.10'
# Run the playbook again in the venv with Jinja < 2.10
ansible-playbook runme.yml "$@"

22
test/integration/targets/filter_urls/runme.sh
View File

@ -1,22 +0,0 @@
#!/usr/bin/env bash
set -eux
export ANSIBLE_ROLES_PATH=../
ansible-playbook runme.yml "$@"
source virtualenv.sh
# This is necessary for installing Jinja 2.6. We need this because Jinja 2.6
# won't install with newer setuptools, and because setuptools 45+ won't work
# with Python 2.
pip install 'setuptools<45'
# Install Jinja 2.6 since we want to test the fallback to Ansible's custom
# urlencode functions. Jinja 2.6 does not have urlencode so we will trigger the
# fallback.
pip install 'jinja2 >= 2.6, < 2.7'
# Run the playbook again in the venv with Jinja 2.6
ansible-playbook runme.yml "$@"

4
test/integration/targets/filter_urls/runme.yml
View File

@ -1,4 +0,0 @@
- hosts: localhost
gather_facts: false
roles:
- { role: filter_urls }

7
test/integration/targets/filter_urls/tasks/main.yml
View File

@ -1,10 +1,3 @@
- name: Get Jinja2 version
shell: "{{ ansible_python_interpreter }} -c 'import jinja2; print(jinja2.__version__)'"
register: jinja2_version
- name: Print Jinja2 version
debug: var=jinja2_version.stdout
- name: Test urldecode filter
set_fact:
urldecoded_string: key="@{}é&%£ foo bar '(;\<>""°)

40
test/integration/targets/find/tasks/main.yml
View File

@ -259,16 +259,10 @@
use_regex: true
exclude: .*\.ogg
register: find_test3
# Note that currently sane ways of doing this with map() or
# selectattr() aren't available in centos6 era jinja2 ...
- set_fact:
find_test3_list: >-
[ {% for f in find_test3.files %}
{{ f.path }}
{% if not loop.last %},{% endif %}
{% endfor %}
]
- debug: var=find_test3_list
find_test3_list: "{{ find_test3.files|map(attribute='path') }}"
- name: assert we skipped the ogg file
assert:
that:
@ -303,12 +297,7 @@
register: result
- set_fact:
astest_list: >-
[ {% for f in result.files %}
{{ f.path }}
{% if not loop.last %},{% endif %}
{% endfor %}
]
astest_list: "{{ result.files|map(attribute='path') }}"
- name: assert we only find the old file
assert:
@ -323,12 +312,7 @@
register: result
- set_fact:
astest_list: >-
[ {% for f in result.files %}
{{ f.path }}
{% if not loop.last %},{% endif %}
{% endfor %}
]
astest_list: "{{ result.files|map(attribute='path') }}"
- name: assert we only find the current file
assert:
@ -348,12 +332,7 @@
register: result
- set_fact:
astest_list: >-
[ {% for f in result.files %}
{{ f.path }}
{% if not loop.last %},{% endif %}
{% endfor %}
]
astest_list: "{{ result.files|map(attribute='path') }}"
- name: assert we only find the hello world file
assert:
@ -372,12 +351,7 @@
register: result
- set_fact:
astest_list: >-
[ {% for f in result.files %}
{{ f.path }}
{% if not loop.last %},{% endif %}
{% endfor %}
]
astest_list: "{{ result.files|map(attribute='path') }}"
- name: assert we do not find the hello world file and a checksum is present
assert:

13
test/integration/targets/git/tasks/archive.yml
View File

@ -11,14 +11,9 @@
register: git_archive
with_items: "{{ git_archive_extensions[ansible_os_family ~ ansible_distribution_major_version | default('default') ] | default(git_archive_extensions.default) }}"
# The map filter was added in Jinja2 2.7, which is newer than the version on RHEL/CentOS 6,
# so we skip this validation on those hosts
- name: ARCHIVE | Assert that archives were downloaded
assert:
that: (git_archive.results | map(attribute='changed') | unique | list)[0]
when:
- "ansible_os_family == 'RedHat'"
- ansible_distribution_major_version is version('7', '>=')
- name: ARCHIVE | Check if archive file is created or not
stat:
@ -53,14 +48,9 @@
register: git_archive
with_items: "{{ git_archive_extensions[ansible_os_family ~ ansible_distribution_major_version | default('default') ] | default(git_archive_extensions.default) }}"
# The map filter was added in Jinja2 2.7, which is newer than the version on RHEL/CentOS 6,
# so we skip this validation on those hosts
- name: ARCHIVE | Assert that archives were downloaded
assert:
that: (git_archive.results | map(attribute='changed') | unique | list)[0]
when:
- "ansible_os_family == 'RedHat'"
- ansible_distribution_major_version is version('7', '>=')
- name: ARCHIVE | Check if archive file is created or not
stat:
@ -82,14 +72,11 @@
register: archive_content
with_items: "{{ git_archive_extensions[ansible_os_family ~ ansible_distribution_major_version | default('default') ] | default(git_archive_extensions.default) }}"
# Does not work on RedHat6 (jinja2 too old?)
- name: ARCHIVE | Ensure archive content is correct
assert:
that:
- item.stdout_lines | sort | first == 'defaults/'
with_items: "{{ archive_content.results }}"
when:
- ansible_os_family ~ ansible_distribution_major_version != 'RedHat6'
- name: ARCHIVE | Clear checkout_dir
file:

1
test/integration/targets/groupby_filter/aliases
View File

@ -1,3 +1,2 @@
shippable/posix/group2
needs/file/test/lib/ansible_test/_data/requirements/constraints.txt
context/controller

4
test/integration/targets/groupby_filter/requirements.txt
View File

@ -1,4 +0,0 @@
# pip 7.1 added support for constraints, which are required by ansible-test to install most python requirements
# see https://github.com/pypa/pip/blame/e648e00dc0226ade30ade99591b245b0c98e86c9/NEWS.rst#L1258
pip >= 7.1, < 10 ; python_version < '2.7' # pip 10+ drops support for python 2.6 (sanity_ok)
pip >= 7.1 ; python_version >= '2.7' # sanity_ok

16
test/integration/targets/groupby_filter/runme.sh
View File

@ -1,16 +0,0 @@
#!/usr/bin/env bash
set -eux
source virtualenv.sh
# Update pip in the venv to a version that supports constraints
pip install --requirement requirements.txt
pip install -U jinja2==2.9.4 --constraint "../../../lib/ansible_test/_data/requirements/constraints.txt"
ansible-playbook -i ../../inventory test_jinja2_groupby.yml -v "$@"
pip install -U "jinja2<2.9.0" --constraint "../../../lib/ansible_test/_data/requirements/constraints.txt"
ansible-playbook -i ../../inventory test_jinja2_groupby.yml -v "$@"

16
test/integration/targets/groupby_filter/tasks/main.yml Normal file
View File

@ -0,0 +1,16 @@
- set_fact:
result: "{{ fruits | groupby('enjoy') }}"
vars:
fruits:
- name: apple
enjoy: yes
- name: orange
enjoy: no
- name: strawberry
enjoy: yes
- assert:
that:
- result == expected
vars:
expected: [[false, [{"enjoy": false, "name": "orange"}]], [true, [{"enjoy": true, "name": "apple"}, {"enjoy": true, "name": "strawberry"}]]]

29
test/integration/targets/groupby_filter/test_jinja2_groupby.yml
View File

@ -1,29 +0,0 @@
---
- name: Test jinja2 groupby
hosts: localhost
gather_facts: True
connection: local
vars:
fruits:
- name: apple
enjoy: yes
- name: orange
enjoy: no
- name: strawberry
enjoy: yes
expected: [[false, [{"enjoy": false, "name": "orange"}]], [true, [{"enjoy": true, "name": "apple"}, {"enjoy": true, "name": "strawberry"}]]]
tasks:
- name: show python interpreter
debug:
msg: "{{ ansible_python['executable'] }}"
- name: show jinja2 version
debug:
msg: "{{ lookup('pipe', '{{ ansible_python[\"executable\"] }} -c \"import jinja2; print(jinja2.__version__)\"') }}"
- set_fact:
result: "{{ fruits | groupby('enjoy') }}"
- assert:
that:
- result == expected

1
test/integration/targets/jinja2_native_types/nested_undefined.yml
View File

@ -21,4 +21,3 @@
- assert:
that:
- "\"'nested_and_undefined' is undefined\" in result.msg"
when: lookup('pipe', ansible_python_interpreter ~ ' -c "import jinja2; print(jinja2.__version__)"') is version('2.10', '>=')

24
test/integration/targets/jinja2_native_types/runtests.yml
View File

@ -31,20 +31,10 @@
s_false: "False"
yaml_none: ~
tasks:
- name: check jinja version
command: "{{ ansible_python_interpreter }} -c 'import jinja2; print(jinja2.__version__)'"
register: jinja2_version
- name: make sure jinja is the right version
set_fact:
is_native: "{{ jinja2_version.stdout is version('2.10', '>=') }}"
- block:
- import_tasks: test_casting.yml
- import_tasks: test_concatentation.yml
- import_tasks: test_bool.yml
- import_tasks: test_dunder.yml
- import_tasks: test_types.yml
- import_tasks: test_none.yml
- import_tasks: test_template.yml
when: is_native
- import_tasks: test_casting.yml
- import_tasks: test_concatentation.yml
- import_tasks: test_bool.yml
- import_tasks: test_dunder.yml
- import_tasks: test_types.yml
- import_tasks: test_none.yml
- import_tasks: test_template.yml

14
test/integration/targets/roles_arg_spec/test_complex_role_fails.yml
View File

@ -38,16 +38,6 @@
]
tasks:
# This test play requires jinja >= 2.7
- name: get the jinja2 version
shell: python -c 'import jinja2; print(jinja2.__version__)'
register: jinja2_version
delegate_to: localhost
changed_when: false
- debug:
msg: "Jinja version: {{ jinja2_version.stdout }}"
- name: include_role test1 since it has a arg_spec.yml
block:
- include_role:
@ -174,7 +164,3 @@
- ansible_failed_result.validate_args_context.name == "test1"
- ansible_failed_result.validate_args_context.type == "role"
- "ansible_failed_result.validate_args_context.path is search('roles_arg_spec/roles/test1')"
# skip this task if jinja isnt >= 2.7, aka centos6
when:
- jinja2_version.stdout is version('2.7', '>=')

8
test/integration/targets/template/72615.yml
View File

@ -16,11 +16,3 @@
- "'top-level-foo' not in template_result"
- "'template-level-foo' in template_result"
- "'template-nested-level-foo' in template_result"
when: lookup('pipe', ansible_python_interpreter ~ ' -c "import jinja2; print(jinja2.__version__)"') is version('2.9', '>=')
- assert:
that:
- "'top-level-foo' in template_result"
- "'template-level-foo' not in template_result"
- "'template-nested-level-foo' not in template_result"
when: lookup('pipe', ansible_python_interpreter ~ ' -c "import jinja2; print(jinja2.__version__)"') is version('2.9', '<')

14
test/integration/targets/template/tasks/main.yml
View File

@ -199,11 +199,6 @@
# VERIFY lstrip_blocks
- name: Check support for lstrip_blocks in Jinja2
shell: "{{ ansible_python.executable }} -c 'import jinja2; jinja2.defaults.LSTRIP_BLOCKS'"
register: lstrip_block_support
ignore_errors: True
- name: Render a template with "lstrip_blocks" set to False
template:
src: lstrip_blocks.j2
@ -229,24 +224,15 @@
register: lstrip_blocks_true_result
ignore_errors: True
- name: Verify exception is thrown if Jinja2 does not support lstrip_blocks but lstrip_blocks is used
assert:
that:
- "lstrip_blocks_true_result.failed"
- 'lstrip_blocks_true_result.msg is search(">=2.7")'
when: "lstrip_block_support is failed"
- name: Get checksum of known good lstrip_blocks_true.expected
stat:
path: "{{role_path}}/files/lstrip_blocks_true.expected"
register: lstrip_blocks_true_good
when: "lstrip_block_support is successful"
- name: Verify templated lstrip_blocks_true matches known good using checksum
assert:
that:
- "lstrip_blocks_true_result.checksum == lstrip_blocks_true_good.stat.checksum"
when: "lstrip_block_support is successful"
# VERIFY CONTENTS

5
test/integration/targets/template_jinja2_latest/aliases
View File

@ -1,5 +0,0 @@
needs/root
shippable/posix/group2
needs/target/template
context/controller
needs/file/test/lib/ansible_test/_data/requirements/constraints.txt

4
test/integration/targets/template_jinja2_latest/main.yml
View File

@ -1,4 +0,0 @@
- hosts: testhost
gather_facts: True
roles:
- { role: template }

4
test/integration/targets/template_jinja2_latest/pip-requirements.txt
View File

@ -1,4 +0,0 @@
# pip 7.1 added support for constraints, which are required by ansible-test to install most python requirements
# see https://github.com/pypa/pip/blame/e648e00dc0226ade30ade99591b245b0c98e86c9/NEWS.rst#L1258
pip >= 7.1, < 10 ; python_version < '2.7' # pip 10+ drops support for python 2.6 (sanity_ok)
pip >= 7.1 ; python_version >= '2.7' # sanity_ok

2
test/integration/targets/template_jinja2_latest/requirements.txt
View File

@ -1,2 +0,0 @@
jinja2 < 2.11 ; python_version < '2.7' # jinja2 2.11 and later require python 2.7 or later
jinja2 ; python_version >= '2.7'

14
test/integration/targets/template_jinja2_latest/runme.sh
View File

@ -1,14 +0,0 @@
#!/usr/bin/env bash
set -eux
source virtualenv.sh
pip install --requirement pip-requirements.txt
pip install -U -r requirements.txt --constraint "../../../lib/ansible_test/_data/requirements/constraints.txt"
ANSIBLE_ROLES_PATH=../
export ANSIBLE_ROLES_PATH
ansible-playbook -i ../../inventory main.yml -v "$@"

1
test/integration/targets/template_jinja2_non_native/46169.yml
View File

@ -29,4 +29,3 @@
- assert:
that:
- native_lookup | type_debug == 'dict'
when: lookup('pipe', ansible_python_interpreter ~ ' -c "import jinja2; print(jinja2.__version__)"') is version('2.10', '>=')

11
test/integration/targets/test_mathstuff/tasks/main.yml
View File

@ -1,8 +1,3 @@
- name: Get Jinja2 version
set_fact:
jinja2_version: >-
{{ lookup('pipe', '{{ ansible_playbook_python }} -c "import jinja2; print(jinja2.__version__)"') }}
- name: Assert subset tests work
assert:
that:
@ -28,11 +23,5 @@
that:
- "'bad' is not nan"
- "1.1 | float is not nan"
# Jinja2 versions prior to 2.10 will traceback when using: 'nan' | float
- name: Assert nan tests work (Jinja2 2.10+)
assert:
that:
- "'nan' | float is isnan" # old name
- "'nan' | float is nan"
when: jinja2_version is version('2.10', '>=')

5
test/integration/targets/unarchive/tasks/test_include.yml
View File

@ -24,17 +24,12 @@
paths: "{{ remote_tmp_dir }}/include-zip"
register: unarchive_dir02
# The map filter was added in Jinja2 2.7, which is newer than the version on RHEL/CentOS 6,
# so we skip this validation on those hosts
- name: Verify that zip extraction included only one file
assert:
that:
- file_names == ['FOO-UNAR.TXT']
vars:
file_names: "{{ unarchive_dir02.files | map(attribute='path') | map('basename') }}"
when:
- "ansible_facts.os_family == 'RedHat'"
- ansible_facts.distribution_major_version is version('7', '>=')
- name: Unpack tar file include one file
unarchive:

3
test/integration/targets/undefined/tasks/main.yml
View File

@ -1,5 +1,4 @@
- when: lookup('pipe', ansible_playbook_python ~ ' -c "import jinja2; print(jinja2.__version__)"') is version('2.7', '>=')
block:
- block:
- set_fact:
names: '{{ things|map(attribute="name") }}'
vars:

2
test/lib/ansible_test/_data/requirements/ansible.txt
View File

@ -3,7 +3,7 @@
# packages. Thus, this should be the loosest set possible (only required
# packages, not optional ones, and with the widest range of versions that could
# be suitable)
jinja2
jinja2 >= 3.0.0
PyYAML
cryptography
packaging

1
test/lib/ansible_test/_internal/ansible_util.py
View File

@ -96,7 +96,6 @@ def ansible_environment(args, color=True, ansible_config=None): # type: (Common
ANSIBLE_CONFIG=ansible_config,
ANSIBLE_LIBRARY='/dev/null',
ANSIBLE_DEVEL_WARNING='false', # Don't show warnings that CI is running devel
ANSIBLE_JINJA2_NATIVE_WARNING='false', # Don't show warnings in CI for old Jinja for native
PYTHONPATH=get_ansible_python_path(args),
PAGER='/bin/cat',
PATH=path,

9
test/lib/ansible_test/_util/target/setup/bootstrap.sh
View File

@ -147,20 +147,18 @@ bootstrap_remote_freebsd()
;;
esac
# Jinja2 is not installed with an OS package since the provided version is too old.
# PyYAML is never installed with an OS package since it does not include libyaml support.
# Instead, ansible-test will install it using pip.
# Instead, ansible-test will install them using pip.
if [ "${have_os_packages}" ]; then
jinja2_pkg="py${python_package_version}-Jinja2"
cryptography_pkg="py${python_package_version}-cryptography"
else
jinja2_pkg=""
cryptography_pkg=""
fi
packages="
${packages}
libyaml
${jinja2_pkg}
${cryptography_pkg}
"
fi
@ -238,10 +236,11 @@ bootstrap_remote_rhel_8()
${py_pkg_prefix}-devel
"
# Jinja2 is not installed with an OS package since the provided version is too old.
# Instead, ansible-test will install it using pip.
if [ "${controller}" ]; then
packages="
${packages}
${py_pkg_prefix}-jinja2
${py_pkg_prefix}-cryptography
"
fi

20
test/units/plugins/filter/test_mathstuff.py
View File

@ -62,26 +62,6 @@ class TestSymmetricDifference:
assert sorted(ms.symmetric_difference(env, tuple(dataset1), tuple(dataset2))) == expected[2]
class TestMin:
def test_min(self):
assert ms.min(env, (1, 2)) == 1
assert ms.min(env, (2, 1)) == 1
assert ms.min(env, ('p', 'a', 'w', 'b', 'p')) == 'a'
assert ms.min(env, ({'key': 'a'}, {'key': 'b'}, {'key': 'c'}), attribute='key') == {'key': 'a'}
assert ms.min(env, ({'key': 1}, {'key': 2}, {'key': 3}), attribute='key') == {'key': 1}
assert ms.min(env, ('a', 'A', 'b', 'B'), case_sensitive=True) == 'A'
class TestMax:
def test_max(self):
assert ms.max(env, (1, 2)) == 2
assert ms.max(env, (2, 1)) == 2
assert ms.max(env, ('p', 'a', 'w', 'b', 'p')) == 'w'
assert ms.max(env, ({'key': 'a'}, {'key': 'b'}, {'key': 'c'}), attribute='key') == {'key': 'c'}
assert ms.max(env, ({'key': 1}, {'key': 2}, {'key': 3}), attribute='key') == {'key': 3}
assert ms.max(env, ('a', 'A', 'b', 'B'), case_sensitive=True) == 'b'
class TestLogarithm:
def test_log_non_number(self):
# Message changed in python3.6

50
test/units/template/test_vars.py
View File

@ -29,53 +29,13 @@ class TestVars(unittest.TestCase):
def setUp(self):
self.mock_templar = MagicMock(name='mock_templar')
def test(self):
ajvars = AnsibleJ2Vars(None, None)
print(ajvars)
def test_globals_empty_2_8(self):
def test_globals_empty(self):
ajvars = AnsibleJ2Vars(self.mock_templar, {})
res28 = self._dict_jinja28(ajvars)
self.assertIsInstance(res28, dict)
res = dict(ajvars)
self.assertIsInstance(res, dict)
def test_globals_empty_2_9(self):
ajvars = AnsibleJ2Vars(self.mock_templar, {})
res29 = self._dict_jinja29(ajvars)
self.assertIsInstance(res29, dict)
def _assert_globals(self, res):
def test_globals(self):
res = dict(AnsibleJ2Vars(self.mock_templar, {'foo': 'bar', 'blip': [1, 2, 3]}))
self.assertIsInstance(res, dict)
self.assertIn('foo', res)
self.assertEqual(res['foo'], 'bar')
def test_globals_2_8(self):
ajvars = AnsibleJ2Vars(self.mock_templar, {'foo': 'bar', 'blip': [1, 2, 3]})
res28 = self._dict_jinja28(ajvars)
self._assert_globals(res28)
def test_globals_2_9(self):
ajvars = AnsibleJ2Vars(self.mock_templar, {'foo': 'bar', 'blip': [1, 2, 3]})
res29 = self._dict_jinja29(ajvars)
self._assert_globals(res29)
def _dicts(self, ajvars):
print(ajvars)
res28 = self._dict_jinja28(ajvars)
res29 = self._dict_jinja29(ajvars)
# res28_other = self._dict_jinja28(ajvars, {'other_key': 'other_value'})
# other = {'other_key': 'other_value'}
# res29_other = self._dict_jinja29(ajvars, *other)
print('res28: %s' % res28)
print('res29: %s' % res29)
# print('res28_other: %s' % res28_other)
# print('res29_other: %s' % res29_other)
# return (res28, res29, res28_other, res29_other)
# assert ajvars == res28
# assert ajvars == res29
return (res28, res29)
def _dict_jinja28(self, *args, **kwargs):
return dict(*args, **kwargs)
def _dict_jinja29(self, the_vars):
return dict(the_vars)