Modernize install (#76021)

Co-authored-by: Matt Clay <matt@mystile.com>
Co-authored-by: Matt Davis <mrd@redhat.com>
Co-authored-by: Sviatoslav Sydorenko <wk.cvs.github@sydorenko.org.ua>

MANIFEST.in

@@ -7,6 +7,10 @@ include docs/docsite/rst/collections/all_plugins.rst
 exclude docs/docsite/rst_warnings
 exclude docs/docsite/rst/conf.py
 exclude docs/docsite/rst/index.rst
+exclude docs/docsite/rst/dev_guide/testing/sanity/bin-symlinks.rst
+exclude docs/docsite/rst/dev_guide/testing/sanity/botmeta.rst
+exclude docs/docsite/rst/dev_guide/testing/sanity/integration-aliases.rst
+exclude docs/docsite/rst/dev_guide/testing/sanity/release-names.rst
 recursive-exclude docs/docsite/_build *
 recursive-exclude docs/docsite/_extensions *.pyc *.pyo
 include examples/hosts
@@ -31,6 +35,9 @@ recursive-include test/lib/ansible_test/_util/controller/sanity/validate-modules
 recursive-include test/sanity *.json *.py *.txt
 recursive-include test/support *.py *.ps1 *.psm1 *.cs
 exclude test/sanity/code-smell/botmeta.*
+exclude test/sanity/code-smell/release-names.*
+exclude test/lib/ansible_test/_internal/commands/sanity/bin_symlinks.py
+exclude test/lib/ansible_test/_internal/commands/sanity/integration_aliases.py
 recursive-include test/units *
 include Makefile
 include MANIFEST.in
@@ -38,3 +45,5 @@ include changelogs/CHANGELOG*.rst
 include changelogs/changelog.yaml
 recursive-include hacking/build_library *.py
 include hacking/build-ansible.py
+include hacking/test-module.py
+include bin/*

bin/ansible

@@ -1 +1 @@
-../lib/ansible/cli/scripts/ansible_cli_stub.py
+../lib/ansible/cli/adhoc.py

bin/ansible-config

@@ -1 +1 @@
-ansible
+../lib/ansible/cli/config.py

bin/ansible-console

@@ -1 +1 @@
-ansible
+../lib/ansible/cli/console.py

bin/ansible-doc

@@ -1 +1 @@
-ansible
+../lib/ansible/cli/doc.py

bin/ansible-galaxy

@@ -1 +1 @@
-ansible
+../lib/ansible/cli/galaxy.py

bin/ansible-inventory

@@ -1 +1 @@
-ansible
+../lib/ansible/cli/inventory.py

bin/ansible-playbook

@@ -1 +1 @@
-ansible
+../lib/ansible/cli/playbook.py

bin/ansible-pull

@@ -1 +1 @@
-ansible
+../lib/ansible/cli/pull.py

bin/ansible-vault

@@ -1 +1 @@
-ansible
+../lib/ansible/cli/vault.py

changelogs/fragments/modernize-install.yaml

@@ -0,0 +1,3 @@
+minor_changes:
+- Installation - modernize our python installation, to reduce dynamic code in setup.py, and migrate
+  what is feasible to setup.cfg. This will enable shipping wheels in the future.

hacking/env-setup

@@ -48,10 +48,12 @@ FULL_PATH=$($PYTHON_BIN -c "import os; print(os.path.realpath('$HACKING_DIR'))")
 export ANSIBLE_HOME="$(dirname "$FULL_PATH")"
 
 PREFIX_PYTHONPATH="$ANSIBLE_HOME/lib"
+ANSIBLE_TEST_PREFIX_PYTHONPATH="$ANSIBLE_HOME/test/lib"
 PREFIX_PATH="$ANSIBLE_HOME/bin"
 PREFIX_MANPATH="$ANSIBLE_HOME/docs/man"
 
 expr "$PYTHONPATH" : "${PREFIX_PYTHONPATH}.*" > /dev/null || prepend_path PYTHONPATH "$PREFIX_PYTHONPATH"
+expr "$PYTHONPATH" : "${ANSIBLE_TEST_PREFIX_PYTHONPATH}.*" > /dev/null || prepend_path PYTHONPATH "$ANSIBLE_TEST_PREFIX_PYTHONPATH"
 expr "$PATH" : "${PREFIX_PATH}.*" > /dev/null || prepend_path PATH "$PREFIX_PATH"
 expr "$MANPATH" : "${PREFIX_MANPATH}.*" > /dev/null || prepend_path MANPATH "$PREFIX_MANPATH"
 

hacking/env-setup.fish

@@ -5,6 +5,7 @@ set HACKING_DIR (dirname (status -f))
 set FULL_PATH (python -c "import os; print(os.path.realpath('$HACKING_DIR'))")
 set ANSIBLE_HOME (dirname $FULL_PATH)
 set PREFIX_PYTHONPATH $ANSIBLE_HOME/lib
+set ANSIBLE_TEST_PREFIX_PYTHONPATH $ANSIBLE_HOME/test/lib
 set PREFIX_PATH $ANSIBLE_HOME/bin
 set PREFIX_MANPATH $ANSIBLE_HOME/docs/man
 
@@ -31,6 +32,16 @@ else
     end
 end
 
+# Set ansible_test PYTHONPATH
+switch PYTHONPATH
+    case "$ANSIBLE_TEST_PREFIX_PYTHONPATH*"
+    case "*"
+        if not [ $QUIET ]
+            echo "Appending PYTHONPATH"
+        end
+        set -gx PYTHONPATH "$ANSIBLE_TEST_PREFIX_PYTHONPATH:$PYTHONPATH"
+end
+
 # Set PATH
 if not contains $PREFIX_PATH $PATH
     set -gx PATH $PREFIX_PATH $PATH

lib/ansible/__main__.py

@@ -0,0 +1,41 @@
+# Copyright: (c) 2021, Matt Martz <matt@sivel.net>
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+import argparse
+import importlib
+import os
+import sys
+
+from importlib.metadata import distribution
+
+
+def _short_name(name):
+    return name.replace('ansible-', '').replace('ansible', 'adhoc')
+
+
+def main():
+    dist = distribution('ansible-core')
+    ep_map = {_short_name(ep.name): ep for ep in dist.entry_points if ep.group == 'console_scripts'}
+
+    parser = argparse.ArgumentParser(prog='python -m ansible', add_help=False)
+    parser.add_argument('entry_point', choices=list(ep_map) + ['test'])
+    args, extra = parser.parse_known_args()
+
+    if args.entry_point == 'test':
+        ansible_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+        source_root = os.path.join(ansible_root, 'test', 'lib')
+
+        if os.path.exists(os.path.join(source_root, 'ansible_test', '_internal', '__init__.py')):
+            # running from source, use that version of ansible-test instead of any version that may already be installed
+            sys.path.insert(0, source_root)
+
+        module = importlib.import_module('ansible_test._util.target.cli.ansible_test_cli_stub')
+        main = module.main
+    else:
+        main = ep_map[args.entry_point].load()
+
+    main([args.entry_point] + extra)
+
+
+if __name__ == '__main__':
+    main()

lib/ansible/cli/__init__.py

@@ -7,17 +7,36 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+import sys
+
+# Used for determining if the system is running a new enough python version
+# and should only restrict on our documented minimum versions
+if sys.version_info < (3, 8):
+    raise SystemExit(
+        'ERROR: Ansible requires Python 3.8 or newer on the controller. '
+        'Current version: %s' % ''.join(sys.version.splitlines())
+    )
+
+import errno
 import getpass
 import os
 import subprocess
-import sys
-
+import traceback
 from abc import ABCMeta, abstractmethod
+from pathlib import Path
+
+try:
+    from ansible import constants as C
+    from ansible.utils.display import Display, initialize_locale
+    initialize_locale()
+    display = Display()
+except Exception as e:
+    print('ERROR: %s' % e, file=sys.stderr)
+    sys.exit(5)
 
-from ansible.cli.arguments import option_helpers as opt_help
-from ansible import constants as C
 from ansible import context
-from ansible.errors import AnsibleError
+from ansible.cli.arguments import option_helpers as opt_help
+from ansible.errors import AnsibleError, AnsibleOptionsError, AnsibleParserError
 from ansible.inventory.manager import InventoryManager
 from ansible.module_utils.six import with_metaclass, string_types, PY3
 from ansible.module_utils._text import to_bytes, to_text
@@ -27,7 +46,6 @@ from ansible.plugins.loader import add_all_plugin_dirs
 from ansible.release import __version__
 from ansible.utils.collection_loader import AnsibleCollectionConfig
 from ansible.utils.collection_loader._collection_finder import _get_collection_name_from_path
-from ansible.utils.display import Display
 from ansible.utils.path import unfrackpath
 from ansible.utils.unsafe_proxy import to_unsafe_text
 from ansible.vars.manager import VariableManager
@@ -39,9 +57,6 @@ except ImportError:
     HAS_ARGCOMPLETE = False
 
 
-display = Display()
-
-
 class CLI(with_metaclass(ABCMeta, object)):
     ''' code behind bin/ansible* programs '''
 
@@ -292,7 +307,7 @@ class CLI(with_metaclass(ABCMeta, object)):
                 ansible.arguments.option_helpers.add_runas_options(self.parser)
                 self.parser.add_option('--my-option', dest='my_option', action='store')
         """
-        self.parser = opt_help.create_base_parser(os.path.basename(self.args[0]), usage=usage, desc=desc, epilog=epilog, )
+        self.parser = opt_help.create_base_parser(self.name, usage=usage, desc=desc, epilog=epilog)
 
     @abstractmethod
     def post_process_args(self, options):
@@ -532,3 +547,74 @@ class CLI(with_metaclass(ABCMeta, object)):
             raise AnsibleError('Empty password was provided from file (%s)' % pwd_file)
 
         return to_unsafe_text(secret)
+
+    @classmethod
+    def cli_executor(cls, args=None):
+        if args is None:
+            args = sys.argv
+
+        try:
+            display.debug("starting run")
+
+            ansible_dir = Path("~/.ansible").expanduser()
+            try:
+                ansible_dir.mkdir(mode=0o700)
+            except OSError as exc:
+                if exc.errno != errno.EEXIST:
+                    display.warning(
+                        "Failed to create the directory '%s': %s" % (ansible_dir, to_text(exc, errors='surrogate_or_replace'))
+                    )
+            else:
+                display.debug("Created the '%s' directory" % ansible_dir)
+
+            try:
+                args = [to_text(a, errors='surrogate_or_strict') for a in args]
+            except UnicodeError:
+                display.error('Command line args are not in utf-8, unable to continue.  Ansible currently only understands utf-8')
+                display.display(u"The full traceback was:\n\n%s" % to_text(traceback.format_exc()))
+                exit_code = 6
+            else:
+                cli = cls(args)
+                exit_code = cli.run()
+
+        except AnsibleOptionsError as e:
+            cli.parser.print_help()
+            display.error(to_text(e), wrap_text=False)
+            exit_code = 5
+        except AnsibleParserError as e:
+            display.error(to_text(e), wrap_text=False)
+            exit_code = 4
+    # TQM takes care of these, but leaving comment to reserve the exit codes
+    #    except AnsibleHostUnreachable as e:
+    #        display.error(str(e))
+    #        exit_code = 3
+    #    except AnsibleHostFailed as e:
+    #        display.error(str(e))
+    #        exit_code = 2
+        except AnsibleError as e:
+            display.error(to_text(e), wrap_text=False)
+            exit_code = 1
+        except KeyboardInterrupt:
+            display.error("User interrupted execution")
+            exit_code = 99
+        except Exception as e:
+            if C.DEFAULT_DEBUG:
+                # Show raw stacktraces in debug mode, It also allow pdb to
+                # enter post mortem mode.
+                raise
+            have_cli_options = bool(context.CLIARGS)
+            display.error("Unexpected Exception, this is probably a bug: %s" % to_text(e), wrap_text=False)
+            if not have_cli_options or have_cli_options and context.CLIARGS['verbosity'] > 2:
+                log_only = False
+                if hasattr(e, 'orig_exc'):
+                    display.vvv('\nexception type: %s' % to_text(type(e.orig_exc)))
+                    why = to_text(e.orig_exc)
+                    if to_text(e) != why:
+                        display.vvv('\noriginal msg: %s' % why)
+            else:
+                display.display("to see the full traceback, use -vvv")
+                log_only = True
+            display.display(u"the full traceback was:\n\n%s" % to_text(traceback.format_exc()), log_only=log_only)
+            exit_code = 250
+
+        sys.exit(exit_code)

lib/ansible/cli/adhoc.py

@@ -1,13 +1,16 @@
+#!/usr/bin/env python
 # Copyright: (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
 # Copyright: (c) 2018, Ansible Project
 # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+# PYTHON_ARGCOMPLETE_OK
 
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+# ansible.cli needs to be imported first, to ensure the source bin/* scripts run that code first
+from ansible.cli import CLI
 from ansible import constants as C
 from ansible import context
-from ansible.cli import CLI
 from ansible.cli.arguments import option_helpers as opt_help
 from ansible.errors import AnsibleError, AnsibleOptionsError
 from ansible.executor.task_queue_manager import TaskQueueManager
@@ -25,6 +28,8 @@ class AdHocCLI(CLI):
         this command allows you to define and run a single task 'playbook' against a set of hosts
     '''
 
+    name = 'ansible'
+
     def init_parser(self):
         ''' create an options parser for bin/ansible '''
         super(AdHocCLI, self).init_parser(usage='%prog <host-pattern> [options]',
@@ -179,3 +184,11 @@ class AdHocCLI(CLI):
                 loader.cleanup_all_tmp_files()
 
         return result
+
+
+def main(args=None):
+    AdHocCLI.cli_executor(args)
+
+
+if __name__ == '__main__':
+    main()

lib/ansible/cli/config.py

@@ -1,9 +1,14 @@
+#!/usr/bin/env python
 # Copyright: (c) 2017, Ansible Project
 # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+# PYTHON_ARGCOMPLETE_OK
 
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+# ansible.cli needs to be imported first, to ensure the source bin/* scripts run that code first
+from ansible.cli import CLI
+
 import os
 import shlex
 import subprocess
@@ -13,7 +18,6 @@ from ansible import context
 import ansible.plugins.loader as plugin_loader
 
 from ansible import constants as C
-from ansible.cli import CLI
 from ansible.cli.arguments import option_helpers as opt_help
 from ansible.config.manager import ConfigManager, Setting
 from ansible.errors import AnsibleError, AnsibleOptionsError
@@ -33,6 +37,8 @@ display = Display()
 class ConfigCLI(CLI):
     """ Config command line class """
 
+    name = 'ansible-config'
+
     def __init__(self, args, callback=None):
 
         self.config_file = None
@@ -469,3 +475,11 @@ class ConfigCLI(CLI):
             text = self._get_plugin_configs(context.CLIARGS['type'], context.CLIARGS['args'])
 
         self.pager(to_text('\n'.join(text), errors='surrogate_or_strict'))
+
+
+def main(args=None):
+    ConfigCLI.cli_executor(args)
+
+
+if __name__ == '__main__':
+    main()

lib/ansible/cli/console.py

@@ -1,11 +1,16 @@
+#!/usr/bin/env python
 # Copyright: (c) 2014, Nandor Sivok <dominis@haxor.hu>
 # Copyright: (c) 2016, Redhat Inc
 # Copyright: (c) 2018, Ansible Project
 # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+# PYTHON_ARGCOMPLETE_OK
 
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+# ansible.cli needs to be imported first, to ensure the source bin/* scripts run that code first
+from ansible.cli import CLI
+
 import atexit
 import cmd
 import getpass
@@ -15,7 +20,6 @@ import sys
 
 from ansible import constants as C
 from ansible import context
-from ansible.cli import CLI
 from ansible.cli.arguments import option_helpers as opt_help
 from ansible.executor.task_queue_manager import TaskQueueManager
 from ansible.module_utils._text import to_native, to_text
@@ -56,6 +60,7 @@ class ConsoleCLI(CLI, cmd.Cmd):
        - `exit`: exit ansible-console
     '''
 
+    name = 'ansible-console'
     modules = []
     ARGUMENTS = {'host-pattern': 'A name of a group in the inventory, a shell-like glob '
                                  'selecting hosts in inventory or any combination of the two separated by commas.'}
@@ -406,7 +411,7 @@ class ConsoleCLI(CLI, cmd.Cmd):
         if module_name in self.modules:
             in_path = module_loader.find_plugin(module_name)
             if in_path:
-                oc, a, _, _ = plugin_docs.get_docstring(in_path, fragment_loader)
+                oc, a, _dummy1, _dummy2 = plugin_docs.get_docstring(in_path, fragment_loader)
                 if oc:
                     display.display(oc['short_description'])
                     display.display('Parameters:')
@@ -438,7 +443,7 @@ class ConsoleCLI(CLI, cmd.Cmd):
 
     def module_args(self, module_name):
         in_path = module_loader.find_plugin(module_name)
-        oc, a, _, _ = plugin_docs.get_docstring(in_path, fragment_loader, is_module=True)
+        oc, a, _dummy1, _dummy2 = plugin_docs.get_docstring(in_path, fragment_loader, is_module=True)
         return list(oc['options'].keys())
 
     def run(self):
@@ -494,3 +499,11 @@ class ConsoleCLI(CLI, cmd.Cmd):
         atexit.register(readline.write_history_file, histfile)
         self.set_prompt()
         self.cmdloop()
+
+
+def main(args=None):
+    ConsoleCLI.cli_executor(args)
+
+
+if __name__ == '__main__':
+    main()

lib/ansible/cli/doc.py

@@ -1,10 +1,15 @@
+#!/usr/bin/env python
 # Copyright: (c) 2014, James Tanner <tanner.jc@gmail.com>
 # Copyright: (c) 2018, Ansible Project
 # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+# PYTHON_ARGCOMPLETE_OK
 
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+# ansible.cli needs to be imported first, to ensure the source bin/* scripts run that code first
+from ansible.cli import CLI
+
 import datetime
 import json
 import pkgutil
@@ -19,7 +24,6 @@ import ansible.plugins.loader as plugin_loader
 
 from ansible import constants as C
 from ansible import context
-from ansible.cli import CLI
 from ansible.cli.arguments import option_helpers as opt_help
 from ansible.collections.list import list_collection_dirs
 from ansible.errors import AnsibleError, AnsibleOptionsError, AnsibleParserError
@@ -335,6 +339,8 @@ class DocCLI(CLI, RoleMixin):
         provides a printout of their DOCUMENTATION strings,
         and it can create a short "snippet" which can be pasted into a playbook.  '''
 
+    name = 'ansible-doc'
+
     # default ignore list for detailed views
     IGNORE = ('module', 'docuri', 'version_added', 'short_description', 'now_date', 'plainexamples', 'returndocs', 'collection')
 
@@ -1383,3 +1389,11 @@ def _do_lookup_snippet(doc):
     text.append(snippet)
 
     return text
+
+
+def main(args=None):
+    DocCLI.cli_executor(args)
+
+
+if __name__ == '__main__':
+    main()

lib/ansible/cli/galaxy.py

@@ -1,10 +1,15 @@
+#!/usr/bin/env python
 # Copyright: (c) 2013, James Cammarata <jcammarata@ansible.com>
 # Copyright: (c) 2018-2021, Ansible Project
 # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+# PYTHON_ARGCOMPLETE_OK
 
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+# ansible.cli needs to be imported first, to ensure the source bin/* scripts run that code first
+from ansible.cli import CLI
+
 import json
 import os.path
 import re
@@ -17,7 +22,6 @@ from yaml.error import YAMLError
 
 import ansible.constants as C
 from ansible import context
-from ansible.cli import CLI
 from ansible.cli.arguments import option_helpers as opt_help
 from ansible.errors import AnsibleError, AnsibleOptionsError
 from ansible.galaxy import Galaxy, get_collections_galaxy_meta_info
@@ -134,6 +138,8 @@ def _get_collection_widths(collections):
 class GalaxyCLI(CLI):
     '''command to manage Ansible roles in shared repositories, the default of which is Ansible Galaxy *https://galaxy.ansible.com*.'''
 
+    name = 'ansible-galaxy'
+
     SKIP_INFO_KEYS = ("name", "description", "readme_html", "related", "summary_fields", "average_aw_composite", "average_aw_score", "url")
 
     def __init__(self, args):
@@ -1679,3 +1685,11 @@ class GalaxyCLI(CLI):
         display.display(resp['status'])
 
         return True
+
+
+def main(args=None):
+    GalaxyCLI.cli_executor(args)
+
+
+if __name__ == '__main__':
+    main()

lib/ansible/cli/inventory.py

@@ -1,10 +1,15 @@
+#!/usr/bin/env python
 # Copyright: (c) 2017, Brian Coca <bcoca@ansible.com>
 # Copyright: (c) 2018, Ansible Project
 # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+# PYTHON_ARGCOMPLETE_OK
 
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+# ansible.cli needs to be imported first, to ensure the source bin/* scripts run that code first
+from ansible.cli import CLI
+
 import sys
 
 import argparse
@@ -12,7 +17,6 @@ from operator import attrgetter
 
 from ansible import constants as C
 from ansible import context
-from ansible.cli import CLI
 from ansible.cli.arguments import option_helpers as opt_help
 from ansible.errors import AnsibleError, AnsibleOptionsError
 from ansible.module_utils._text import to_bytes, to_native, to_text
@@ -46,6 +50,8 @@ INTERNAL_VARS = frozenset(['ansible_diff_mode',
 class InventoryCLI(CLI):
     ''' used to display or dump the configured inventory as Ansible sees it '''
 
+    name = 'ansible-inventory'
+
     ARGUMENTS = {'host': 'The name of a host to match in the inventory, relevant when using --list',
                  'group': 'The name of a group in the inventory, relevant when using --graph', }
 
@@ -402,3 +408,11 @@ class InventoryCLI(CLI):
         results = format_group(top)
 
         return results
+
+
+def main(args=None):
+    InventoryCLI.cli_executor(args)
+
+
+if __name__ == '__main__':
+    main()

lib/ansible/cli/playbook.py

@@ -1,16 +1,20 @@
+#!/usr/bin/env python
 # (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
 # Copyright: (c) 2018, Ansible Project
 # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+# PYTHON_ARGCOMPLETE_OK
 
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+# ansible.cli needs to be imported first, to ensure the source bin/* scripts run that code first
+from ansible.cli import CLI
+
 import os
 import stat
 
 from ansible import constants as C
 from ansible import context
-from ansible.cli import CLI
 from ansible.cli.arguments import option_helpers as opt_help
 from ansible.errors import AnsibleError
 from ansible.executor.playbook_executor import PlaybookExecutor
@@ -29,6 +33,8 @@ class PlaybookCLI(CLI):
     ''' the tool to run *Ansible playbooks*, which are a configuration and multinode deployment system.
         See the project home page (https://docs.ansible.com) for more information. '''
 
+    name = 'ansible-playbook'
+
     def init_parser(self):
 
         # create parser for CLI options
@@ -215,3 +221,11 @@ class PlaybookCLI(CLI):
         for host in inventory.list_hosts():
             hostname = host.get_name()
             variable_manager.clear_facts(hostname)
+
+
+def main(args=None):
+    PlaybookCLI.cli_executor(args)
+
+
+if __name__ == '__main__':
+    main()

lib/ansible/cli/pull.py

@@ -1,10 +1,15 @@
+#!/usr/bin/env python
 # Copyright: (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
 # Copyright: (c) 2018, Ansible Project
 # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+# PYTHON_ARGCOMPLETE_OK
 
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+# ansible.cli needs to be imported first, to ensure the source bin/* scripts run that code first
+from ansible.cli import CLI
+
 import datetime
 import os
 import platform
@@ -16,7 +21,6 @@ import time
 
 from ansible import constants as C
 from ansible import context
-from ansible.cli import CLI
 from ansible.cli.arguments import option_helpers as opt_help
 from ansible.errors import AnsibleOptionsError
 from ansible.module_utils._text import to_native, to_text
@@ -40,6 +44,8 @@ class PullCLI(CLI):
         excellent way to gather and analyze remote logs from ansible-pull.
     '''
 
+    name = 'ansible-pull'
+
     DEFAULT_REPO_TYPE = 'git'
     DEFAULT_PLAYBOOK = 'local.yml'
     REPO_CHOICES = ('git', 'subversion', 'hg', 'bzr')
@@ -341,3 +347,11 @@ class PullCLI(CLI):
             if playbook is None:
                 display.warning("\n".join(errors))
             return playbook
+
+
+def main(args=None):
+    PullCLI.cli_executor(args)
+
+
+if __name__ == '__main__':
+    main()

lib/ansible/cli/scripts/ansible_cli_stub.py

@@ -1,170 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
-#
-# This file is part of Ansible
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-
-# PYTHON_ARGCOMPLETE_OK
-
-from __future__ import (absolute_import, division, print_function)
-__metaclass__ = type
-
-
-import errno
-import os
-import shutil
-import sys
-import traceback
-
-# Used for determining if the system is running a new enough python version
-# and should only restrict on our documented minimum versions
-_PY38_MIN = sys.version_info[:2] >= (3, 8)
-if not _PY38_MIN:
-    raise SystemExit(
-        'ERROR: Ansible requires Python 3.8 or newer on the controller. '
-        'Current version: %s' % ''.join(sys.version.splitlines())
-    )
-
-
-# These lines appear after the PY38 check, to ensure the "friendly" error happens before
-# any invalid syntax appears in other files that may get imported
-from ansible import context
-from ansible.errors import AnsibleError, AnsibleOptionsError, AnsibleParserError
-from ansible.module_utils._text import to_text
-
-from pathlib import Path
-
-
-class LastResort(object):
-    # OUTPUT OF LAST RESORT
-    def display(self, msg, log_only=None):
-        print(msg, file=sys.stderr)
-
-    def error(self, msg, wrap_text=None):
-        print(msg, file=sys.stderr)
-
-
-if __name__ == '__main__':
-
-    display = LastResort()
-
-    try:  # bad ANSIBLE_CONFIG or config options can force ugly stacktrace
-        import ansible.constants as C
-        from ansible.utils.display import Display, initialize_locale
-    except AnsibleOptionsError as e:
-        display.error(to_text(e), wrap_text=False)
-        sys.exit(5)
-
-    initialize_locale()
-
-    cli = None
-    me = Path(sys.argv[0]).name
-
-    try:
-        display = Display()
-        display.debug("starting run")
-
-        sub = None
-        target = me.split('-')
-        if target[-1][0].isdigit():
-            # Remove any version or python version info as downstreams
-            # sometimes add that
-            target = target[:-1]
-
-        if len(target) > 1:
-            sub = target[1]
-            myclass = "%sCLI" % sub.capitalize()
-        elif target[0] == 'ansible':
-            sub = 'adhoc'
-            myclass = 'AdHocCLI'
-        else:
-            raise AnsibleError("Unknown Ansible alias: %s" % me)
-
-        try:
-            mycli = getattr(__import__("ansible.cli.%s" % sub, fromlist=[myclass]), myclass)
-        except ImportError as e:
-            # ImportError members have changed in py3
-            if 'msg' in dir(e):
-                msg = e.msg
-            else:
-                msg = e.message
-            if msg.endswith(' %s' % sub):
-                raise AnsibleError("Ansible sub-program not implemented: %s" % me)
-            else:
-                raise
-
-        ansible_dir = Path("~/.ansible").expanduser()
-        try:
-            ansible_dir.mkdir(mode=0o700)
-        except OSError as exc:
-            if exc.errno != errno.EEXIST:
-                display.warning(
-                    "Failed to create the directory '%s': %s" % (ansible_dir, to_text(exc, errors='surrogate_or_replace'))
-                )
-        else:
-            display.debug("Created the '%s' directory" % ansible_dir)
-
-        try:
-            args = [to_text(a, errors='surrogate_or_strict') for a in sys.argv]
-        except UnicodeError:
-            display.error('Command line args are not in utf-8, unable to continue.  Ansible currently only understands utf-8')
-            display.display(u"The full traceback was:\n\n%s" % to_text(traceback.format_exc()))
-            exit_code = 6
-        else:
-            cli = mycli(args)
-            exit_code = cli.run()
-
-    except AnsibleOptionsError as e:
-        cli.parser.print_help()
-        display.error(to_text(e), wrap_text=False)
-        exit_code = 5
-    except AnsibleParserError as e:
-        display.error(to_text(e), wrap_text=False)
-        exit_code = 4
-# TQM takes care of these, but leaving comment to reserve the exit codes
-#    except AnsibleHostUnreachable as e:
-#        display.error(str(e))
-#        exit_code = 3
-#    except AnsibleHostFailed as e:
-#        display.error(str(e))
-#        exit_code = 2
-    except AnsibleError as e:
-        display.error(to_text(e), wrap_text=False)
-        exit_code = 1
-    except KeyboardInterrupt:
-        display.error("User interrupted execution")
-        exit_code = 99
-    except Exception as e:
-        if C.DEFAULT_DEBUG:
-            # Show raw stacktraces in debug mode, It also allow pdb to
-            # enter post mortem mode.
-            raise
-        have_cli_options = bool(context.CLIARGS)
-        display.error("Unexpected Exception, this is probably a bug: %s" % to_text(e), wrap_text=False)
-        if not have_cli_options or have_cli_options and context.CLIARGS['verbosity'] > 2:
-            log_only = False
-            if hasattr(e, 'orig_exc'):
-                display.vvv('\nexception type: %s' % to_text(type(e.orig_exc)))
-                why = to_text(e.orig_exc)
-                if to_text(e) != why:
-                    display.vvv('\noriginal msg: %s' % why)
-        else:
-            display.display("to see the full traceback, use -vvv")
-            log_only = True
-        display.display(u"the full traceback was:\n\n%s" % to_text(traceback.format_exc()), log_only=log_only)
-        exit_code = 250
-
-    sys.exit(exit_code)

lib/ansible/cli/scripts/ansible_connection_cli_stub.py

@@ -6,6 +6,7 @@ from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 
+import argparse
 import fcntl
 import hashlib
 import os
@@ -20,6 +21,7 @@ import json
 from contextlib import contextmanager
 
 from ansible import constants as C
+from ansible.cli.arguments.option_helpers import AnsibleVersion
 from ansible.module_utils._text import to_bytes, to_text
 from ansible.module_utils.six import PY3
 from ansible.module_utils.six.moves import cPickle, StringIO
@@ -32,6 +34,8 @@ from ansible.utils.path import unfrackpath, makedirs_safe
 from ansible.utils.display import Display
 from ansible.utils.jsonrpc import JsonRpcServer
 
+display = Display()
+
 
 def read_stream(byte_stream):
     size = int(byte_stream.readline().strip())
@@ -217,9 +221,15 @@ class ConnectionProcess(object):
         display.display('shutdown complete', log_only=True)
 
 
-def main():
+def main(args=None):
     """ Called to initiate the connect to the remote device
     """
+    parser = argparse.ArgumentParser(prog='ansible-connection', add_help=False)
+    parser.add_argument('--version', action=AnsibleVersion, nargs=0)
+    parser.add_argument('playbook_pid')
+    parser.add_argument('task_uuid')
+    args = parser.parse_args(args[1:] if args is not None else args)
+
     rc = 0
     result = {}
     messages = list()
@@ -260,8 +270,8 @@ def main():
 
     if rc == 0:
         ssh = connection_loader.get('ssh', class_only=True)
-        ansible_playbook_pid = sys.argv[1]
-        task_uuid = sys.argv[2]
+        ansible_playbook_pid = args.playbook_pid
+        task_uuid = args.task_uuid
         cp = ssh._create_control_path(play_context.remote_addr, play_context.port, play_context.remote_user, play_context.connection, ansible_playbook_pid)
         # create the persistent connection dir if need be and create the paths
         # which we will be using later
@@ -345,5 +355,4 @@ def main():
 
 
 if __name__ == '__main__':
-    display = Display()
     main()

lib/ansible/cli/vault.py

@@ -1,16 +1,20 @@
+#!/usr/bin/env python
 # (c) 2014, James Tanner <tanner.jc@gmail.com>
 # Copyright: (c) 2018, Ansible Project
 # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+# PYTHON_ARGCOMPLETE_OK
 
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+# ansible.cli needs to be imported first, to ensure the source bin/* scripts run that code first
+from ansible.cli import CLI
+
 import os
 import sys
 
 from ansible import constants as C
 from ansible import context
-from ansible.cli import CLI
 from ansible.cli.arguments import option_helpers as opt_help
 from ansible.errors import AnsibleOptionsError
 from ansible.module_utils._text import to_text, to_bytes
@@ -32,6 +36,8 @@ class VaultCLI(CLI):
     If you'd like to not expose what variables you are using, you can keep an individual task file entirely encrypted.
     '''
 
+    name = 'ansible-vault'
+
     FROM_STDIN = "stdin"
     FROM_ARGS = "the command line args"
     FROM_PROMPT = "the interactive prompt"
@@ -462,3 +468,11 @@ class VaultCLI(CLI):
                                    self.new_encrypt_vault_id)
 
         display.display("Rekey successful", stderr=True)
+
+
+def main(args=None):
+    VaultCLI.cli_executor(args)
+
+
+if __name__ == '__main__':
+    main()

pyproject.toml

@@ -0,0 +1,3 @@
+[build-system]
+requires = ["setuptools >= 39.2.0", "wheel"]
+build-backend = "setuptools.build_meta"

setup.cfg

@@ -0,0 +1,60 @@
+# Minimum target setuptools 39.2.0
+
+[metadata]
+name = ansible-core
+version = attr: ansible.release.__version__
+description = Radically simple IT automation
+long_description = file: README.rst
+author = Ansible, Inc.
+author_email = info@ansible.com
+url = https://ansible.com/
+project_urls =
+    Bug Tracker=https://github.com/ansible/ansible/issues
+    CI: Azure Pipelines=https://dev.azure.com/ansible/ansible/
+    Code of Conduct=https://docs.ansible.com/ansible/latest/community/code_of_conduct.html
+    Documentation=https://docs.ansible.com/ansible-core/
+    Mailing lists=https://docs.ansible.com/ansible/latest/community/communication.html#mailing-list-information
+    Source Code=https://github.com/ansible/ansible
+license = GPLv3+
+classifiers =
+    Development Status :: 5 - Production/Stable
+    Environment :: Console
+    Intended Audience :: Developers
+    Intended Audience :: Information Technology
+    Intended Audience :: System Administrators
+    License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
+    Natural Language :: English
+    Operating System :: POSIX
+    Programming Language :: Python :: 3
+    Programming Language :: Python :: 3.8
+    Programming Language :: Python :: 3.9
+    Programming Language :: Python :: 3.10
+    Programming Language :: Python :: 3 :: Only
+    Topic :: System :: Installation/Setup
+    Topic :: System :: Systems Administration
+    Topic :: Utilities
+
+[options]
+zip_safe = False
+python_requires = >=3.8
+include_package_data = True
+# keep ansible-test as a verbatim script to work with editable installs, since it needs to do its
+# own package redirection magic that's beyond the scope of the normal `ansible` path redirection
+# done by setuptools `develop`
+scripts =
+    bin/ansible-test
+
+# setuptools 51.0.0
+# [options.entry_points]
+# console_scripts =
+#     ansible = ansible.cli.adhoc:main
+#     ansible-config = ansible.cli.config:main
+#     ansible-console = ansible.cli.console:main
+#     ansible-doc = ansible.cli.doc:main
+#     ansible-galaxy = ansible.cli.galaxy:main
+#     ansible-inventory = ansible.cli.inventory:main
+#     ansible-playbook = ansible.cli.playbook:main
+#     ansible-pull = ansible.cli.pull:main
+#     ansible-vault = ansible.cli.vault:main
+#     ansible-connection = ansible.cli.scripts.ansible_connection_cli_stub:main
+#     ansible-test = ansible_test._util.target.cli.ansible_test_cli_stub:main

setup.py

@@ -1,394 +1,31 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-import json
-import os
-import os.path
-import re
-import sys
-import warnings
+import pathlib
 
-from collections import defaultdict
+from setuptools import find_packages, setup
 
-try:
-    from setuptools import setup, find_packages
-    from setuptools.command.build_py import build_py as BuildPy
-    from setuptools.command.install_lib import install_lib as InstallLib
-    from setuptools.command.install_scripts import install_scripts as InstallScripts
-except ImportError:
-    print("Ansible now needs setuptools in order to build. Install it using"
-          " your package manager (usually python-setuptools) or via pip (pip"
-          " install setuptools).", file=sys.stderr)
-    sys.exit(1)
+here = pathlib.Path(__file__).parent.resolve()
 
-# `distutils` must be imported after `setuptools` or it will cause explosions
-# with `setuptools >=48.0.0, <49.1`.
-# Refs:
-# * https://github.com/ansible/ansible/issues/70456
-# * https://github.com/pypa/setuptools/issues/2230
-# * https://github.com/pypa/setuptools/commit/bd110264
-from distutils.command.build_scripts import build_scripts as BuildScripts
-from distutils.command.sdist import sdist as SDist
+install_requires = (here / 'requirements.txt').read_text(encoding='utf-8').splitlines()
 
-
-def find_package_info(*file_paths):
-    try:
-        with open(os.path.join(*file_paths), 'r') as f:
-            info_file = f.read()
-    except Exception:
-        raise RuntimeError("Unable to find package info.")
-
-    # The version line must have the form
-    # __version__ = 'ver'
-    version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]",
-                              info_file, re.M)
-    author_match = re.search(r"^__author__ = ['\"]([^'\"]*)['\"]",
-                             info_file, re.M)
-
-    if version_match and author_match:
-        return version_match.group(1), author_match.group(1)
-    raise RuntimeError("Unable to find package info.")
-
-
-def _validate_install_ansible_core():
-    """Validate that we can install ansible-core. This checks if
-    ansible<=2.9 or ansible-base>=2.10 are installed.
-    """
-    # Skip common commands we can ignore
-    # Do NOT add bdist_wheel here, we don't ship wheels
-    # and bdist_wheel is the only place we can prevent pip
-    # from installing, as pip creates a wheel, and installs the wheel
-    # and we have no influence over installation within a wheel
-    if set(('sdist', 'egg_info')).intersection(sys.argv):
-        return
-
-    if os.getenv('ANSIBLE_SKIP_CONFLICT_CHECK', '') not in ('', '0'):
-        return
-
-    # Save these for later restoring things to pre invocation
-    sys_modules = sys.modules.copy()
-    sys_modules_keys = set(sys_modules)
-
-    # Make sure `lib` isn't in `sys.path` that could confuse this
-    sys_path = sys.path[:]
-    abspath = os.path.abspath
-    sys.path[:] = [p for p in sys.path if abspath(p) != abspath('lib')]
-
-    try:
-        from ansible.release import __version__
-    except ImportError:
-        pass
-    else:
-        version_tuple = tuple(int(v) for v in __version__.split('.')[:2])
-        if version_tuple >= (2, 11):
-            return
-        elif version_tuple == (2, 10):
-            ansible_name = 'ansible-base'
-        else:
-            ansible_name = 'ansible'
-
-        stars = '*' * 76
-        raise RuntimeError(
-            '''
-
-    %s
-
-    Cannot install ansible-core with a pre-existing %s==%s
-    installation.
-
-    Installing ansible-core with ansible-2.9 or older, or ansible-base-2.10
-    currently installed with pip is known to cause problems. Please uninstall
-    %s and install the new version:
-
-        pip uninstall %s
-        pip install ansible-core
-
-    If you want to skip the conflict checks and manually resolve any issues
-    afterwards, set the ANSIBLE_SKIP_CONFLICT_CHECK environment variable:
-
-        ANSIBLE_SKIP_CONFLICT_CHECK=1 pip install ansible-core
-
-    %s
-            ''' % (stars, ansible_name, __version__, ansible_name, ansible_name, stars))
-    finally:
-        sys.path[:] = sys_path
-        for key in sys_modules_keys.symmetric_difference(sys.modules):
-            sys.modules.pop(key, None)
-        sys.modules.update(sys_modules)
-
-
-_validate_install_ansible_core()
-
-
-SYMLINK_CACHE = 'SYMLINK_CACHE.json'
-
-
-def _find_symlinks(topdir, extension=''):
-    """Find symlinks that should be maintained
-
-    Maintained symlinks exist in the bin dir or are modules which have
-    aliases.  Our heuristic is that they are a link in a certain path which
-    point to a file in the same directory.
-
-    .. warn::
-
-        We want the symlinks in :file:`bin/` that link into :file:`lib/ansible/*` (currently,
-        :command:`ansible`, :command:`ansible-test`, and :command:`ansible-connection`) to become
-        real files on install.  Updates to the heuristic here *must not* add them to the symlink
-        cache.
-    """
-    symlinks = defaultdict(list)
-    for base_path, dirs, files in os.walk(topdir):
-        for filename in files:
-            filepath = os.path.join(base_path, filename)
-            if os.path.islink(filepath) and filename.endswith(extension):
-                target = os.readlink(filepath)
-                if target.startswith('/'):
-                    # We do not support absolute symlinks at all
-                    continue
-
-                if os.path.dirname(target) == '':
-                    link = filepath[len(topdir):]
-                    if link.startswith('/'):
-                        link = link[1:]
-                    symlinks[os.path.basename(target)].append(link)
-                else:
-                    # Count how many directory levels from the topdir we are
-                    levels_deep = os.path.dirname(filepath).count('/')
-
-                    # Count the number of directory levels higher we walk up the tree in target
-                    target_depth = 0
-                    for path_component in target.split('/'):
-                        if path_component == '..':
-                            target_depth += 1
-                            # If we walk past the topdir, then don't store
-                            if target_depth >= levels_deep:
-                                break
-                        else:
-                            target_depth -= 1
-                    else:
-                        # If we managed to stay within the tree, store the symlink
-                        link = filepath[len(topdir):]
-                        if link.startswith('/'):
-                            link = link[1:]
-                        symlinks[target].append(link)
-
-    return symlinks
-
-
-def _cache_symlinks(symlink_data):
-    with open(SYMLINK_CACHE, 'w') as f:
-        json.dump(symlink_data, f)
-
-
-def _maintain_symlinks(symlink_type, base_path):
-    """Switch a real file into a symlink"""
-    try:
-        # Try the cache first because going from git checkout to sdist is the
-        # only time we know that we're going to cache correctly
-        with open(SYMLINK_CACHE, 'r') as f:
-            symlink_data = json.load(f)
-    except (IOError, OSError) as e:
-        # IOError on py2, OSError on py3.  Both have errno
-        if e.errno == 2:
-            # SYMLINKS_CACHE doesn't exist.  Fallback to trying to create the
-            # cache now.  Will work if we're running directly from a git
-            # checkout or from an sdist created earlier.
-            library_symlinks = _find_symlinks('lib', '.py')
-            library_symlinks.update(_find_symlinks('test/lib'))
-
-            symlink_data = {'script': _find_symlinks('bin'),
-                            'library': library_symlinks,
-                            }
-
-            # Sanity check that something we know should be a symlink was
-            # found.  We'll take that to mean that the current directory
-            # structure properly reflects symlinks in the git repo
-            if 'ansible-playbook' in symlink_data['script']['ansible']:
-                _cache_symlinks(symlink_data)
-            else:
-                raise RuntimeError(
-                    "Pregenerated symlink list was not present and expected "
-                    "symlinks in ./bin were missing or broken. "
-                    "Perhaps this isn't a git checkout?"
-                )
-        else:
-            raise
-    symlinks = symlink_data[symlink_type]
-
-    for source in symlinks:
-        for dest in symlinks[source]:
-            dest_path = os.path.join(base_path, dest)
-            if not os.path.islink(dest_path):
-                try:
-                    os.unlink(dest_path)
-                except OSError as e:
-                    if e.errno == 2:
-                        # File does not exist which is all we wanted
-                        pass
-                os.symlink(source, dest_path)
-
-
-class BuildPyCommand(BuildPy):
-    def run(self):
-        BuildPy.run(self)
-        _maintain_symlinks('library', self.build_lib)
-
-
-class BuildScriptsCommand(BuildScripts):
-    def run(self):
-        BuildScripts.run(self)
-        _maintain_symlinks('script', self.build_dir)
-
-
-class InstallLibCommand(InstallLib):
-    def run(self):
-        InstallLib.run(self)
-        _maintain_symlinks('library', self.install_dir)
-
-
-class InstallScriptsCommand(InstallScripts):
-    def run(self):
-        InstallScripts.run(self)
-        _maintain_symlinks('script', self.install_dir)
-
-
-class SDistCommand(SDist):
-    def run(self):
-        # have to generate the cache of symlinks for release as sdist is the
-        # only command that has access to symlinks from the git repo
-        library_symlinks = _find_symlinks('lib', '.py')
-        library_symlinks.update(_find_symlinks('test/lib'))
-
-        symlinks = {'script': _find_symlinks('bin'),
-                    'library': library_symlinks,
-                    }
-        _cache_symlinks(symlinks)
-
-        SDist.run(self)
-
-        # Print warnings at the end because no one will see warnings before all the normal status
-        # output
-        if os.environ.get('_ANSIBLE_SDIST_FROM_MAKEFILE', False) != '1':
-            warnings.warn('When setup.py sdist is run from outside of the Makefile,'
-                          ' the generated tarball may be incomplete.  Use `make snapshot`'
-                          ' to create a tarball from an arbitrary checkout or use'
-                          ' `cd packaging/release && make release version=[..]` for official builds.',
-                          RuntimeWarning)
-
-
-def read_file(file_name):
-    """Read file and return its contents."""
-    with open(file_name, 'r') as f:
-        return f.read()
-
-
-def read_requirements(file_name):
-    """Read requirements file as a list."""
-    reqs = read_file(file_name).splitlines()
-    if not reqs:
-        raise RuntimeError(
-            "Unable to read requirements from the %s file"
-            "That indicates this copy of the source code is incomplete."
-            % file_name
-        )
-    return reqs
-
-
-def get_dynamic_setup_params():
-    """Add dynamically calculated setup params to static ones."""
-    return {
-        # Retrieve the long description from the README
-        'long_description': read_file('README.rst'),
-        'install_requires': read_requirements('requirements.txt'),
-    }
-
-
-here = os.path.abspath(os.path.dirname(__file__))
-__version__, __author__ = find_package_info(here, 'lib', 'ansible', 'release.py')
-static_setup_params = dict(
-    # Use the distutils SDist so that symlinks are not expanded
-    # Use a custom Build for the same reason
-    cmdclass={
-        'build_py': BuildPyCommand,
-        'build_scripts': BuildScriptsCommand,
-        'install_lib': InstallLibCommand,
-        'install_scripts': InstallScriptsCommand,
-        'sdist': SDistCommand,
-    },
-    name='ansible-core',
-    version=__version__,
-    description='Radically simple IT automation',
-    author=__author__,
-    author_email='info@ansible.com',
-    url='https://ansible.com/',
-    project_urls={
-        'Bug Tracker': 'https://github.com/ansible/ansible/issues',
-        'CI: Azure Pipelines': 'https://dev.azure.com/ansible/ansible/',
-        'Code of Conduct': 'https://docs.ansible.com/ansible/latest/community/code_of_conduct.html',
-        'Documentation': 'https://docs.ansible.com/ansible/',
-        'Mailing lists': 'https://docs.ansible.com/ansible/latest/community/communication.html#mailing-list-information',
-        'Source Code': 'https://github.com/ansible/ansible',
-    },
-    license='GPLv3+',
-    # Ansible will also make use of a system copy of python-six and
-    # python-selectors2 if installed but use a Bundled copy if it's not.
-    python_requires='>=3.8',
+setup(
+    install_requires=install_requires,
     package_dir={'': 'lib',
                  'ansible_test': 'test/lib/ansible_test'},
     packages=find_packages('lib') + find_packages('test/lib'),
-    include_package_data=True,
-    classifiers=[
-        'Development Status :: 5 - Production/Stable',
-        'Environment :: Console',
-        'Intended Audience :: Developers',
-        'Intended Audience :: Information Technology',
-        'Intended Audience :: System Administrators',
-        'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)',
-        'Natural Language :: English',
-        'Operating System :: POSIX',
-        'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.8',
-        'Programming Language :: Python :: 3.9',
-        'Programming Language :: Python :: 3.10',
-        'Topic :: System :: Installation/Setup',
-        'Topic :: System :: Systems Administration',
-        'Topic :: Utilities',
-    ],
-    scripts=[
-        'bin/ansible',
-        'bin/ansible-playbook',
-        'bin/ansible-pull',
-        'bin/ansible-doc',
-        'bin/ansible-galaxy',
-        'bin/ansible-console',
-        'bin/ansible-connection',
-        'bin/ansible-vault',
-        'bin/ansible-config',
-        'bin/ansible-inventory',
-        'bin/ansible-test',
-    ],
-    data_files=[],
-    # Installing as zip files would break due to references to __file__
-    zip_safe=False
+    entry_points={
+        'console_scripts': [
+            'ansible=ansible.cli.adhoc:main',
+            'ansible-config=ansible.cli.config:main',
+            'ansible-console=ansible.cli.console:main',
+            'ansible-doc=ansible.cli.doc:main',
+            'ansible-galaxy=ansible.cli.galaxy:main',
+            'ansible-inventory=ansible.cli.inventory:main',
+            'ansible-playbook=ansible.cli.playbook:main',
+            'ansible-pull=ansible.cli.pull:main',
+            'ansible-vault=ansible.cli.vault:main',
+            'ansible-connection=ansible.cli.scripts.ansible_connection_cli_stub:main',
+        ],
+    },
 )
-
-
-def main():
-    """Invoke installation process using setuptools."""
-    setup_params = dict(static_setup_params, **get_dynamic_setup_params())
-    ignore_warning_regex = (
-        r"Unknown distribution option: '(project_urls|python_requires)'"
-    )
-    warnings.filterwarnings(
-        'ignore',
-        message=ignore_warning_regex,
-        category=UserWarning,
-        module='distutils.dist',
-    )
-    setup(**setup_params)
-    warnings.resetwarnings()
-
-
-if __name__ == '__main__':
-    main()

test/integration/targets/connection_local/aliases

@@ -1 +1,2 @@
 shippable/posix/group3
+needs/target/connection

test/integration/targets/connection_local/runme.sh

@@ -1 +0,0 @@
-../connection_posix/test.sh

test/integration/targets/connection_local/runme.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+set -eux
+
+group=local
+
+cd ../connection
+
+INVENTORY="../connection_${group}/test_connection.inventory" ./test.sh \
+    -e target_hosts="${group}" \
+    -e action_prefix= \
+    -e local_tmp=/tmp/ansible-local \
+    -e remote_tmp=/tmp/ansible-remote \
+    "$@"

test/integration/targets/connection_paramiko_ssh/aliases

@@ -1,4 +1,5 @@
 needs/ssh
 shippable/posix/group3
 needs/target/setup_paramiko
+needs/target/connection
 destructive  # potentially installs/uninstalls OS packages via setup_paramiko

test/integration/targets/connection_paramiko_ssh/test.sh

@@ -1 +0,0 @@
-../connection_posix/test.sh

test/integration/targets/connection_paramiko_ssh/test.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+set -eux
+
+group=paramiko_ssh
+
+cd ../connection
+
+INVENTORY="../connection_${group}/test_connection.inventory" ./test.sh \
+    -e target_hosts="${group}" \
+    -e action_prefix= \
+    -e local_tmp=/tmp/ansible-local \
+    -e remote_tmp=/tmp/ansible-remote \
+    "$@"

test/integration/targets/connection_posix/aliases

@@ -1,2 +0,0 @@
-needs/target/connection
-hidden

test/integration/targets/connection_posix/test.sh

@@ -1,18 +0,0 @@
-#!/usr/bin/env bash
-
-set -eux
-
-# Connection tests for POSIX platforms use this script by linking to it from the appropriate 'connection_' target dir.
-# The name of the inventory group to test is extracted from the directory name following the 'connection_' prefix.
-
-group=$(python -c \
-    "from os import path; print(path.basename(path.abspath(path.dirname('$0'))).replace('connection_', ''))")
-
-cd ../connection
-
-INVENTORY="../connection_${group}/test_connection.inventory" ./test.sh \
-    -e target_hosts="${group}" \
-    -e action_prefix= \
-    -e local_tmp=/tmp/ansible-local \
-    -e remote_tmp=/tmp/ansible-remote \
-    "$@"

test/integration/targets/connection_ssh/aliases

@@ -1,2 +1,3 @@
 needs/ssh
 shippable/posix/group1
+needs/target/connection

test/integration/targets/connection_ssh/posix.sh

@@ -1 +0,0 @@
-../connection_posix/test.sh

test/integration/targets/connection_ssh/posix.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+set -eux
+
+group=ssh
+
+cd ../connection
+
+INVENTORY="../connection_${group}/test_connection.inventory" ./test.sh \
+    -e target_hosts="${group}" \
+    -e action_prefix= \
+    -e local_tmp=/tmp/ansible-local \
+    -e remote_tmp=/tmp/ansible-remote \
+    "$@"

test/integration/targets/copy/files/subdir/subdir1/bar.txt

@@ -1 +0,0 @@
-../bar.txt

test/integration/targets/copy/tasks/main.yml

@@ -16,6 +16,7 @@
           invalid2: ../invalid
           out_of_tree_circle: /tmp/ansible-test-link-dir/out_of_tree_circle
           subdir3: ../subdir2/subdir3
+          bar.txt: ../bar.txt
 
     - file: path={{local_temp_dir}} state=directory
       name: ensure temp dir exists

test/integration/targets/entry_points/aliases

@@ -0,0 +1,2 @@
+context/controller
+shippable/posix/group5

test/integration/targets/entry_points/runme.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+set -eu
+source virtualenv.sh
+set +x
+unset PYTHONPATH
+
+base_dir="$(dirname "$(dirname "$(dirname "$(dirname "${OUTPUT_DIR}")")")")"
+bin_dir="$(dirname "$(command -v pip)")"
+
+# deps are already installed, using --no-deps to avoid re-installing them
+pip install "${base_dir}" --disable-pip-version-check --no-deps
+# --use-feature=in-tree-build not available on all platforms
+
+for bin in "${bin_dir}/ansible"*; do
+    name="$(basename "${bin}")"
+
+    entry_point="${name//ansible-/}"
+    entry_point="${entry_point//ansible/adhoc}"
+
+    echo "=== ${name} (${entry_point})=${bin} ==="
+
+    if [ "${name}" == "ansible-test" ]; then
+        "${bin}" --help | tee /dev/stderr | grep -Eo "^usage:\ ansible-test\ .*"
+        python -m ansible "${entry_point}" --help | tee /dev/stderr | grep -Eo "^usage:\ ansible-test\ .*"
+    else
+        "${bin}" --version | tee /dev/stderr | grep -Eo "(^${name}\ \[core\ .*|executable location = ${bin}$)"
+        python -m ansible "${entry_point}" --version | tee /dev/stderr | grep -Eo "(^${name}\ \[core\ .*|executable location = ${bin}$)"
+    fi
+done

test/integration/targets/plugin_loader/normal/library/_symlink.py

@@ -1 +0,0 @@
-_underscore.py

test/integration/targets/plugin_loader/runme.sh

@@ -2,6 +2,15 @@
 
 set -ux
 
+cleanup() {
+    unlink normal/library/_symlink.py
+}
+
+pushd normal/library
+ln -s _underscore.py _symlink.py
+popd
+
+trap 'cleanup' EXIT
 
 # check normal execution
 for myplay in normal/*.yml

test/lib/ansible_test/_internal/__init__.py

@@ -47,11 +47,11 @@ from .provisioning import (
 )
 
 
-def main():
+def main(cli_args=None):  # type: (t.Optional[t.List[str]]) -> None
     """Main program function."""
     try:
         os.chdir(data_context().content.root)
-        args = parse_args()
+        args = parse_args(cli_args)
         config = args.config(args)  # type: CommonConfig
         display.verbosity = config.verbosity
         display.truncate = config.truncate

test/lib/ansible_test/_internal/cli/__init__.py

@@ -20,7 +20,7 @@ from .compat import (
 )
 
 
-def parse_args():  # type: () -> argparse.Namespace
+def parse_args(argv=None):  # type: (t.Optional[t.List[str]]) -> argparse.Namespace
     """Parse command line arguments."""
     completer = CompositeActionCompletionFinder()
 
@@ -29,7 +29,7 @@ def parse_args():  # type: () -> argparse.Namespace
     else:
         epilog = 'Install the "argcomplete" python package to enable tab completion.'
 
-    parser = argparse.ArgumentParser(epilog=epilog)
+    parser = argparse.ArgumentParser(prog='ansible-test', epilog=epilog)
 
     do_commands(parser, completer)
 
@@ -38,7 +38,10 @@ def parse_args():  # type: () -> argparse.Namespace
         always_complete_options=False,
     )
 
-    argv = sys.argv[1:]
+    if argv is None:
+        argv = sys.argv[1:]
+    else:
+        argv = argv[1:]
     args = parser.parse_args(argv)
 
     if args.explain and not args.verbosity:

test/lib/ansible_test/_internal/constants.py

@@ -35,15 +35,15 @@ SECCOMP_CHOICES = [
 # It is necessary for payload creation to reconstruct the bin directory when running ansible-test from an installed version of ansible.
 # It is also used to construct the injector directory at runtime.
 ANSIBLE_BIN_SYMLINK_MAP = {
-    'ansible': '../lib/ansible/cli/scripts/ansible_cli_stub.py',
-    'ansible-config': 'ansible',
+    'ansible': '../lib/ansible/cli/adhoc.py',
+    'ansible-config': '../lib/ansible/cli/config.py',
     'ansible-connection': '../lib/ansible/cli/scripts/ansible_connection_cli_stub.py',
-    'ansible-console': 'ansible',
-    'ansible-doc': 'ansible',
-    'ansible-galaxy': 'ansible',
-    'ansible-inventory': 'ansible',
-    'ansible-playbook': 'ansible',
-    'ansible-pull': 'ansible',
+    'ansible-console': '../lib/ansible/cli/console.py',
+    'ansible-doc': '../lib/ansible/cli/doc.py',
+    'ansible-galaxy': '../lib/ansible/cli/galaxy.py',
+    'ansible-inventory': '../lib/ansible/cli/inventory.py',
+    'ansible-playbook': '../lib/ansible/cli/playbook.py',
+    'ansible-pull': '../lib/ansible/cli/pull.py',
     'ansible-test': '../test/lib/ansible_test/_util/target/cli/ansible_test_cli_stub.py',
-    'ansible-vault': 'ansible',
+    'ansible-vault': '../lib/ansible/cli/vault.py',
 }

test/lib/ansible_test/_util/controller/sanity/code-smell/shebang.py

@@ -70,6 +70,10 @@ def main():
                 is_module = True
             elif path == 'test/lib/ansible_test/_util/target/cli/ansible_test_cli_stub.py':
                 pass  # ansible-test entry point must be executable and have a shebang
+            elif re.search(r'^lib/ansible/cli/[^/]+\.py', path):
+                pass  # cli entry points must be executable and have a shebang
+            elif path.startswith('examples/'):
+                continue  # examples trigger some false positives due to location
             elif path.startswith('lib/') or path.startswith('test/lib/'):
                 if executable:
                     print('%s:%d:%d: should not be executable' % (path, 0, 0))

test/lib/ansible_test/_util/target/cli/ansible_test_cli_stub.py

@@ -11,7 +11,7 @@ import os
 import sys
 
 
-def main():
+def main(args=None):  # type: (t.Optional[t.List[str]]) -> None
     """Main program entry point."""
     ansible_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
     source_root = os.path.join(ansible_root, 'test', 'lib')
@@ -30,7 +30,7 @@ def main():
     # noinspection PyProtectedMember
     from ansible_test._internal import main as cli_main
 
-    cli_main()
+    cli_main(args)
 
 
 def version_to_str(version):

test/sanity/code-smell/package-data.py

@@ -29,6 +29,7 @@ def assemble_files_to_ship(complete_file_list):
         'hacking/tests/*',
         'hacking/ticket_stubs/*',
         'test/sanity/code-smell/botmeta.*',
+        'test/sanity/code-smell/release-names.*',
         'test/utils/*',
         'test/utils/*/*',
         'test/utils/*/*/*',
@@ -53,8 +54,9 @@ def assemble_files_to_ship(complete_file_list):
         'hacking/report.py',
         'hacking/return_skeleton_generator.py',
         'hacking/test-module',
-        'hacking/test-module.py',
         'test/support/README.md',
+        'test/lib/ansible_test/_internal/commands/sanity/bin_symlinks.py',
+        'test/lib/ansible_test/_internal/commands/sanity/integration_aliases.py',
         '.cherry_picker.toml',
         '.mailmap',
         # Generated as part of a build step
@@ -74,22 +76,27 @@ def assemble_files_to_ship(complete_file_list):
         'hacking/env-setup',
         'hacking/env-setup.fish',
         'MANIFEST',
+        'setup.cfg',
+        # docs for test files not included in sdist
+        'docs/docsite/rst/dev_guide/testing/sanity/bin-symlinks.rst',
+        'docs/docsite/rst/dev_guide/testing/sanity/botmeta.rst',
+        'docs/docsite/rst/dev_guide/testing/sanity/integration-aliases.rst',
+        'docs/docsite/rst/dev_guide/testing/sanity/release-names.rst',
     ))
 
     # These files are generated and then intentionally added to the sdist
 
     # Manpages
+    ignore_script = ('ansible-connection', 'ansible-test')
     manpages = ['docs/man/man1/ansible.1']
     for dirname, dummy, files in os.walk('bin'):
         for filename in files:
-            path = os.path.join(dirname, filename)
-            if os.path.islink(path):
-                if os.readlink(path) == 'ansible':
-                    manpages.append('docs/man/man1/%s.1' % filename)
+            if filename in ignore_script:
+                continue
+            manpages.append('docs/man/man1/%s.1' % filename)
 
     # Misc
     misc_generated_files = [
-        'SYMLINK_CACHE.json',
         'PKG-INFO',
     ]
 
@@ -110,7 +117,11 @@ def assemble_files_to_install(complete_file_list):
     """
     This looks for all of the files which should show up in an installation of ansible
     """
-    ignore_patterns = tuple()
+    ignore_patterns = (
+        # Tests excluded from sdist
+        'test/lib/ansible_test/_internal/commands/sanity/bin_symlinks.py',
+        'test/lib/ansible_test/_internal/commands/sanity/integration_aliases.py',
+    )
 
     pkg_data_files = []
     for path in complete_file_list:
@@ -256,12 +267,19 @@ def check_sdist_files_are_wanted(sdist_dir, to_ship_files):
             dirname = ''
 
         for filename in files:
+            if filename == 'setup.cfg':
+                continue
+
             path = os.path.join(dirname, filename)
             if path not in to_ship_files:
+
                 if fnmatch.fnmatch(path, 'changelogs/CHANGELOG-v2.[0-9]*.rst'):
                     # changelog files are expected
                     continue
 
+                if fnmatch.fnmatch(path, 'lib/ansible_core.egg-info/*'):
+                    continue
+
                 # FIXME: ansible-test doesn't pass the paths of symlinks to us so we aren't
                 # checking those
                 if not os.path.islink(os.path.join(sdist_dir, path)):
@@ -282,7 +300,7 @@ def check_installed_contains_expected(install_dir, to_install_files):
 
 
 EGG_RE = re.compile('ansible[^/]+\\.egg-info/(PKG-INFO|SOURCES.txt|'
-                    'dependency_links.txt|not-zip-safe|requires.txt|top_level.txt)$')
+                    'dependency_links.txt|not-zip-safe|requires.txt|top_level.txt|entry_points.txt)$')
 
 
 def check_installed_files_are_wanted(install_dir, to_install_files):

test/sanity/ignore.txt

@@ -1,14 +1,8 @@
 docs/docsite/rst/dev_guide/testing/sanity/no-smart-quotes.rst no-smart-quotes
 docs/docsite/rst/locales/ja/LC_MESSAGES/dev_guide.po no-smart-quotes  # Translation of the no-smart-quotes rule
-examples/play.yml shebang
 examples/scripts/ConfigureRemotingForAnsible.ps1 pslint:PSCustomUseLiteralPath
-examples/scripts/my_test_facts.py shebang # example module but not in a normal module location
-examples/scripts/my_test_info.py shebang # example module but not in a normal module location
-examples/scripts/my_test.py shebang # example module but not in a normal module location
 examples/scripts/upgrade_to_ps3.ps1 pslint:PSCustomUseLiteralPath
 examples/scripts/upgrade_to_ps3.ps1 pslint:PSUseApprovedVerbs
-lib/ansible/cli/console.py pylint:disallowed-name
-lib/ansible/cli/scripts/ansible_cli_stub.py shebang
 lib/ansible/cli/scripts/ansible_connection_cli_stub.py shebang
 lib/ansible/config/base.yml no-unwanted-files
 lib/ansible/executor/playbook_executor.py pylint:disallowed-name