internet/community.general
3
0
Fork 0
mirror of https://github.com/ansible-collections/community.general.git synced 2026-05-07 13:52:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Performance improvements

Browse Source 
* Speed up serialization of hostvars by simply using the internal
  dictionary used for cached lookups
* Use blocking gets/puts on queues instead of spin locking
* Merge sequential implicitly created blocks
This commit is contained in:
James Cammarata
2015-08-17 13:45:07 -04:00
parent 1d402ae390
commit e7b2308b66
5 changed files with 57 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
lib/ansible/executor/process/result.py
View File

@@ -72,9 +72,9 @@ class ResultProcess(multiprocessing.Process):
self._cur_worker = 0
try:
if not rslt_q.empty():
if rslt_q.qsize() > 0:
debug("worker %d has data to read" % self._cur_worker)
result = rslt_q.get(block=False)
result = rslt_q.get()
debug("got a result from worker %d: %s" % (self._cur_worker, result))
break
except queue.Empty:
@@ -102,7 +102,7 @@ class ResultProcess(multiprocessing.Process):
try:
result = self._read_worker_result()
if result is None:
time.sleep(0.1)
time.sleep(0.01)
continue
# if this task is registering a result, do it now

54
lib/ansible/executor/process/worker.py
View File

@@ -92,40 +92,36 @@ class WorkerProcess(multiprocessing.Process):
while True:
task = None
try:
if not self._main_q.empty():
debug("there's work to be done!")
(host, task, basedir, job_vars, play_context, shared_loader_obj) = self._main_q.get(block=False)
debug("got a task/handler to work on: %s" % task)
(host, task, basedir, job_vars, play_context, shared_loader_obj) = self._main_q.get()
debug("there's work to be done!")
debug("got a task/handler to work on: %s" % task)
# because the task queue manager starts workers (forks) before the
# playbook is loaded, set the basedir of the loader inherted by
# this fork now so that we can find files correctly
self._loader.set_basedir(basedir)
# because the task queue manager starts workers (forks) before the
# playbook is loaded, set the basedir of the loader inherted by
# this fork now so that we can find files correctly
self._loader.set_basedir(basedir)
# Serializing/deserializing tasks does not preserve the loader attribute,
# since it is passed to the worker during the forking of the process and
# would be wasteful to serialize. So we set it here on the task now, and
# the task handles updating parent/child objects as needed.
task.set_loader(self._loader)
# Serializing/deserializing tasks does not preserve the loader attribute,
# since it is passed to the worker during the forking of the process and
# would be wasteful to serialize. So we set it here on the task now, and
# the task handles updating parent/child objects as needed.
task.set_loader(self._loader)
# apply the given task's information to the connection info,
# which may override some fields already set by the play or
# the options specified on the command line
new_play_context = play_context.set_task_and_variable_override(task=task, variables=job_vars)
# apply the given task's information to the connection info,
# which may override some fields already set by the play or
# the options specified on the command line
new_play_context = play_context.set_task_and_variable_override(task=task, variables=job_vars)
# execute the task and build a TaskResult from the result
debug("running TaskExecutor() for %s/%s" % (host, task))
executor_result = TaskExecutor(host, task, job_vars, new_play_context, self._new_stdin, self._loader, shared_loader_obj).run()
debug("done running TaskExecutor() for %s/%s" % (host, task))
task_result = TaskResult(host, task, executor_result)
# execute the task and build a TaskResult from the result
debug("running TaskExecutor() for %s/%s" % (host, task))
executor_result = TaskExecutor(host, task, job_vars, new_play_context, self._new_stdin, self._loader, shared_loader_obj).run()
debug("done running TaskExecutor() for %s/%s" % (host, task))
task_result = TaskResult(host, task, executor_result)
# put the result on the result queue
debug("sending task result")
self._rslt_q.put(task_result, block=False)
debug("done sending task result")
else:
time.sleep(0.1)
# put the result on the result queue
debug("sending task result")
self._rslt_q.put(task_result)
debug("done sending task result")
except queue.Empty:
pass