internet/community.general
3
0
Fork 0
mirror of https://github.com/ansible-collections/community.general.git synced 2026-05-08 06:12:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fact collector ordering deps (#31362)

Browse Source 
Add deps/requires for fact collectors

Fact collectors can now set a required_facts
class attribute that will be a set of the names
of fact collectors they require to be run first.

ie, if a collector needs to know the ansible_distribution,
it should set it's required_facts to include 'distribution'

        required_facts = set(['distribution'])

If a collector requires another collector, it gets added
to the selected collector names.

We then topological sort the ordering of the collectors
so that deps work out (ie, 'distribution' will run before
'service_mgr')

required_facts were added to the collectors for:

        - network (requires 'distribution', 'platform')
        - hardware (requires 'platform')
        - service_mgr (requires 'distribution', 'platform')

Fix name references for facts (need 'ansible_' prefix)
is service_mgr

Fixes #30753
This commit is contained in:
Adrian Likins
2018-01-22 18:23:40 -05:00
committed by GitHub
parent 5fa29201a7
commit 6e585bdf24
12 changed files with 439 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
test/integration/targets/gathering_facts/cache_plugins/none.py Normal file
View File

@@ -0,0 +1,50 @@
# (c) 2014, Brian Coca, Josh Drake, et al
# (c) 2017 Ansible Project
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
from ansible.plugins.cache import BaseCacheModule
DOCUMENTATION = '''
cache: none
short_description: write-only cache (no cache)
description:
- No caching at all
version_added: historical
author: core team (@ansible-core)
'''
class CacheModule(BaseCacheModule):
def __init__(self, *args, **kwargs):
self.empty = {}
def get(self, key):
return self.empty.get(key)
def set(self, key, value):
return value
def keys(self):
return self.empty.keys()
def contains(self, key):
return key in self.empty
def delete(self, key):
del self.emtpy[key]
def flush(self):
self.empty = {}
def copy(self):
return self.empty.copy()
def __getstate__(self):
return self.copy()
def __setstate__(self, data):
self.empty = data

2
test/integration/targets/gathering_facts/runme.sh
View File

@@ -2,4 +2,6 @@
set -eux
# ANSIBLE_CACHE_PLUGINS=cache_plugins/ ANSIBLE_CACHE_PLUGIN=none ansible-playbook test_gathering_facts.yml -i ../../inventory -v "$@"
ansible-playbook test_gathering_facts.yml -i ../../inventory -v "$@"
#ANSIBLE_CACHE_PLUGIN=base ansible-playbook test_gathering_facts.yml -i ../../inventory -v "$@"

8
test/integration/targets/gathering_facts/test_gathering_facts.yml
View File

@@ -112,7 +112,7 @@
tasks:
- setup:
filter: "*env*"
register: fact_results
# register: fact_results
- name: Test that retrieving all facts filtered to env works
assert:
@@ -129,7 +129,7 @@
tasks:
- setup:
filter: "ansible_user_id"
register: fact_results
# register: fact_results
- name: Test that retrieving all facts filtered to specific fact ansible_user_id works
assert:
@@ -148,7 +148,7 @@
tasks:
- setup:
filter: "*"
register: fact_results
# register: fact_results
- name: Test that retrieving all facts filtered to splat
assert:
@@ -165,7 +165,7 @@
tasks:
- setup:
filter: ""
register: fact_results
# register: fact_results
- name: Test that retrieving all facts filtered to empty filter_spec works
assert:

278
test/units/module_utils/facts/test_collector.py
View File

@@ -21,6 +21,7 @@ from __future__ import (absolute_import, division)
__metaclass__ = type
from collections import defaultdict
import pprint
# for testing
from ansible.compat.tests import unittest
@@ -54,34 +55,22 @@ class TestFindCollectorsForPlatform(unittest.TestCase):
class TestSelectCollectorNames(unittest.TestCase):
def _assert_equal_detail(self, obj1, obj2, msg=None):
msg = 'objects are not equal\n%s\n\n!=\n\n%s' % (pprint.pformat(obj1), pprint.pformat(obj2))
return self.assertEqual(obj1, obj2, msg)
def test(self):
collector_names = set(['distribution', 'all_ipv4_addresses',
'local', 'pkg_mgr'])
collector_names = ['distribution', 'all_ipv4_addresses',
'local', 'pkg_mgr']
all_fact_subsets = self._all_fact_subsets()
all_collector_classes = self._all_collector_classes()
res = collector.select_collector_classes(collector_names,
all_fact_subsets,
all_collector_classes)
all_fact_subsets)
expected = [default_collectors.DistributionFactCollector,
default_collectors.PkgMgrFactCollector]
self.assertEqual(res, expected)
def test_reverse(self):
collector_names = set(['distribution', 'all_ipv4_addresses',
'local', 'pkg_mgr'])
all_fact_subsets = self._all_fact_subsets()
all_collector_classes = self._all_collector_classes()
all_collector_classes.reverse()
res = collector.select_collector_classes(collector_names,
all_fact_subsets,
all_collector_classes)
expected = [default_collectors.PkgMgrFactCollector,
default_collectors.DistributionFactCollector]
self.assertEqual(res, expected)
self._assert_equal_detail(res, expected)
def test_default_collectors(self):
platform_info = {'system': 'Generic'}
@@ -95,14 +84,22 @@ class TestSelectCollectorNames(unittest.TestCase):
collector_names = collector.get_collector_names(valid_subsets=all_valid_subsets,
aliases_map=aliases_map,
platform_info=platform_info)
collector.select_collector_classes(collector_names,
all_fact_subsets,
default_collectors.collectors)
complete_collector_names = collector._solve_deps(collector_names, all_fact_subsets)
def _all_collector_classes(self):
return [default_collectors.DistributionFactCollector,
default_collectors.PkgMgrFactCollector,
default_collectors.LinuxNetworkCollector]
dep_map = collector.build_dep_data(complete_collector_names, all_fact_subsets)
ordered_deps = collector.tsort(dep_map)
ordered_collector_names = [x[0] for x in ordered_deps]
res = collector.select_collector_classes(ordered_collector_names,
all_fact_subsets)
self.assertTrue(res.index(default_collectors.ServiceMgrFactCollector) >
res.index(default_collectors.DistributionFactCollector),
res)
self.assertTrue(res.index(default_collectors.ServiceMgrFactCollector) >
res.index(default_collectors.PlatformFactCollector),
res)
def _all_fact_subsets(self, data=None):
all_fact_subsets = defaultdict(list)
@@ -276,30 +273,249 @@ class TestGetCollectorNames(unittest.TestCase):
gather_subset=['my_fact', 'not_a_valid_gather_subset'])
class TestFindUnresolvedRequires(unittest.TestCase):
def test(self):
names = ['network', 'virtual', 'env']
all_fact_subsets = {'env': [default_collectors.EnvFactCollector],
'network': [default_collectors.LinuxNetworkCollector],
'virtual': [default_collectors.LinuxVirtualCollector]}
res = collector.find_unresolved_requires(names, all_fact_subsets)
# pprint.pprint(res)
self.assertIsInstance(res, set)
self.assertEqual(res, set(['platform', 'distribution']))
def test_resolved(self):
names = ['network', 'virtual', 'env', 'platform', 'distribution']
all_fact_subsets = {'env': [default_collectors.EnvFactCollector],
'network': [default_collectors.LinuxNetworkCollector],
'distribution': [default_collectors.DistributionFactCollector],
'platform': [default_collectors.PlatformFactCollector],
'virtual': [default_collectors.LinuxVirtualCollector]}
res = collector.find_unresolved_requires(names, all_fact_subsets)
# pprint.pprint(res)
self.assertIsInstance(res, set)
self.assertEqual(res, set())
class TestBuildDepData(unittest.TestCase):
def test(self):
names = ['network', 'virtual', 'env']
all_fact_subsets = {'env': [default_collectors.EnvFactCollector],
'network': [default_collectors.LinuxNetworkCollector],
'virtual': [default_collectors.LinuxVirtualCollector]}
res = collector.build_dep_data(names, all_fact_subsets)
# pprint.pprint(dict(res))
self.assertIsInstance(res, defaultdict)
self.assertEqual(dict(res),
{'network': set(['platform', 'distribution']),
'virtual': set(),
'env': set()})
class TestSolveDeps(unittest.TestCase):
def test_no_solution(self):
unresolved = set(['required_thing1', 'required_thing2'])
all_fact_subsets = {'env': [default_collectors.EnvFactCollector],
'network': [default_collectors.LinuxNetworkCollector],
'virtual': [default_collectors.LinuxVirtualCollector]}
self.assertRaises(collector.CollectorNotFoundError,
collector._solve_deps,
unresolved,
all_fact_subsets)
def test(self):
unresolved = set(['env', 'network'])
all_fact_subsets = {'env': [default_collectors.EnvFactCollector],
'network': [default_collectors.LinuxNetworkCollector],
'virtual': [default_collectors.LinuxVirtualCollector],
'platform': [default_collectors.PlatformFactCollector],
'distribution': [default_collectors.DistributionFactCollector]}
res = collector.resolve_requires(unresolved, all_fact_subsets)
res = collector._solve_deps(unresolved, all_fact_subsets)
self.assertIsInstance(res, set)
for goal in unresolved:
self.assertIn(goal, res)
class TestResolveRequires(unittest.TestCase):
def test_no_resolution(self):
unresolved = ['required_thing1', 'required_thing2']
all_fact_subsets = {'env': [default_collectors.EnvFactCollector],
'network': [default_collectors.LinuxNetworkCollector],
'virtual': [default_collectors.LinuxVirtualCollector]}
self.assertRaisesRegexp(collector.UnresolvedFactDep,
'unresolved fact dep.*required_thing2',
collector.resolve_requires,
unresolved, all_fact_subsets)
def test(self):
unresolved = ['env', 'network']
all_fact_subsets = {'env': [default_collectors.EnvFactCollector],
'network': [default_collectors.LinuxNetworkCollector],
'virtual': [default_collectors.LinuxVirtualCollector]}
res = collector.resolve_requires(unresolved, all_fact_subsets)
for goal in unresolved:
self.assertIn(goal, res)
def test_exception(self):
unresolved = ['required_thing1']
all_fact_subsets = {}
try:
collector.resolve_requires(unresolved, all_fact_subsets)
except collector.UnresolvedFactDep as exc:
self.assertIn(unresolved[0], '%s' % exc)
class TestTsort(unittest.TestCase):
def test(self):
dep_map = {'network': set(['distribution', 'platform']),
'virtual': set(),
'platform': set(['what_platform_wants']),
'what_platform_wants': set(),
'network_stuff': set(['network'])}
res = collector.tsort(dep_map)
# pprint.pprint(res)
self.assertIsInstance(res, list)
names = [x[0] for x in res]
self.assertTrue(names.index('network_stuff') > names.index('network'))
self.assertTrue(names.index('platform') > names.index('what_platform_wants'))
self.assertTrue(names.index('network') > names.index('platform'))
def test_cycles(self):
dep_map = {'leaf1': set(),
'leaf2': set(),
'node1': set(['node2']),
'node2': set(['node3']),
'node3': set(['node1'])}
self.assertRaises(collector.CycleFoundInFactDeps,
collector.tsort,
dep_map)
def test_just_nodes(self):
dep_map = {'leaf1': set(),
'leaf4': set(),
'leaf3': set(),
'leaf2': set()}
res = collector.tsort(dep_map)
self.assertIsInstance(res, list)
names = [x[0] for x in res]
# not a lot to assert here, any order of the
# results is valid
self.assertEqual(set(names), set(dep_map.keys()))
def test_self_deps(self):
dep_map = {'node1': set(['node1']),
'node2': set(['node2'])}
self.assertRaises(collector.CycleFoundInFactDeps,
collector.tsort,
dep_map)
def test_unsolvable(self):
dep_map = {'leaf1': set(),
'node2': set(['leaf2'])}
res = collector.tsort(dep_map)
self.assertIsInstance(res, list)
names = [x[0] for x in res]
self.assertEqual(set(names), set(dep_map.keys()))
def test_chain(self):
dep_map = {'leaf1': set(['leaf2']),
'leaf2': set(['leaf3']),
'leaf3': set(['leaf4']),
'leaf4': set(),
'leaf5': set(['leaf1'])}
res = collector.tsort(dep_map)
self.assertIsInstance(res, list)
names = [x[0] for x in res]
self.assertEqual(set(names), set(dep_map.keys()))
def test_multi_pass(self):
dep_map = {'leaf1': set(),
'leaf2': set(['leaf3', 'leaf1', 'leaf4', 'leaf5']),
'leaf3': set(['leaf4', 'leaf1']),
'leaf4': set(['leaf1']),
'leaf5': set(['leaf1'])}
res = collector.tsort(dep_map)
self.assertIsInstance(res, list)
names = [x[0] for x in res]
self.assertEqual(set(names), set(dep_map.keys()))
self.assertTrue(names.index('leaf1') < names.index('leaf2'))
for leaf in ('leaf2', 'leaf3', 'leaf4', 'leaf5'):
self.assertTrue(names.index('leaf1') < names.index(leaf))
class TestCollectorClassesFromGatherSubset(unittest.TestCase):
maxDiff = None
def _classes(self,
all_collector_classes=None,
valid_subsets=None,
minimal_gather_subset=None,
gather_subset=None,
gather_timeout=None):
gather_timeout=None,
platform_info=None):
platform_info = platform_info or {'system': 'Linux'}
return collector.collector_classes_from_gather_subset(all_collector_classes=all_collector_classes,
valid_subsets=valid_subsets,
minimal_gather_subset=minimal_gather_subset,
gather_subset=gather_subset,
gather_timeout=gather_timeout)
gather_timeout=gather_timeout,
platform_info=platform_info)
def test_no_args(self):
res = self._classes()
self.assertIsInstance(res, list)
self.assertEqual(res, [])
def test(self):
def test_not_all(self):
res = self._classes(all_collector_classes=default_collectors.collectors,
gather_subset=['!all'])
self.assertIsInstance(res, list)
self.assertEqual(res, [])
def test_all(self):
res = self._classes(all_collector_classes=default_collectors.collectors,
gather_subset=['all'])
self.assertIsInstance(res, list)
def test_hardware(self):
res = self._classes(all_collector_classes=default_collectors.collectors,
gather_subset=['hardware'])
self.assertIsInstance(res, list)
self.assertIn(default_collectors.PlatformFactCollector, res)
self.assertIn(default_collectors.LinuxHardwareCollector, res)
self.assertTrue(res.index(default_collectors.LinuxHardwareCollector) >
res.index(default_collectors.PlatformFactCollector))
def test_network(self):
res = self._classes(all_collector_classes=default_collectors.collectors,
gather_subset=['network'])
self.assertIsInstance(res, list)
self.assertIn(default_collectors.DistributionFactCollector, res)
self.assertIn(default_collectors.PlatformFactCollector, res)
self.assertIn(default_collectors.LinuxNetworkCollector, res)
self.assertTrue(res.index(default_collectors.LinuxNetworkCollector) >
res.index(default_collectors.PlatformFactCollector))
self.assertTrue(res.index(default_collectors.LinuxNetworkCollector) >
res.index(default_collectors.DistributionFactCollector))
# self.assertEqual(set(res, [default_collectors.DistributionFactCollector,
# default_collectors.PlatformFactCollector,
# default_collectors.LinuxNetworkCollector])
def test_env(self):
res = self._classes(all_collector_classes=default_collectors.collectors,
gather_subset=['env'])