ansible-test - Handle externally managed Python (#82883)

Remove EXTERNALLY-MANAGED marker in ansible-test managed environments

Additional changes:

- Test the Alpine 3.19 container
- Stop testing Alpine 3.18 container and remote
- Add missing changelog entry the Alpine 3.19 container

.azure-pipelines/azure-pipelines.yml

@@ -108,8 +108,6 @@ stages:
       - template: templates/matrix.yml  # context/controller (ansible-test container management)
         parameters:
           targets:
-            - name: Alpine 3.18
-              test: alpine/3.18
             - name: Alpine 3.19
               test: alpine/3.19
             - name: Fedora 39
@@ -127,8 +125,8 @@ stages:
         parameters:
           testFormat: linux/{0}
           targets:
-            - name: Alpine 3
-              test: alpine3
+            - name: Alpine 3.19
+              test: alpine319
             - name: Fedora 39
               test: fedora39
             - name: Ubuntu 20.04
@@ -142,8 +140,8 @@ stages:
         parameters:
           testFormat: linux/{0}
           targets:
-            - name: Alpine 3
-              test: alpine3
+            - name: Alpine 3.19
+              test: alpine319
             - name: Fedora 39
               test: fedora39
             - name: Ubuntu 22.04

changelogs/fragments/ansible-test-externally-managed-python.yml

@@ -0,0 +1,4 @@
+minor_changes:
+  - ansible-test - Containers and remotes managed by ansible-test will have their Python ``EXTERNALLY-MANAGED`` marker (PEP668) removed.
+    This provides backwards compatibility for existing tests running in newer environments which mark their Python as externally managed.
+    A future version of ansible-test may change this behavior, requiring tests to be adapted to such environments.

changelogs/fragments/ansible_test_alpine_3.19.yml

@@ -1,2 +1,3 @@
 minor_changes:
   - ansible-test - Add Alpine 3.19 to remotes.
+  - ansible-test - Add Alpine 3.19 container.

test/lib/ansible_test/_internal/bootstrap.py

@@ -28,7 +28,7 @@ class Bootstrap:
     """Base class for bootstrapping systems."""
 
     controller: bool
-    python_versions: list[str]
+    python_interpreters: dict[str, str]
     ssh_key: SshKey
 
     @property
@@ -41,7 +41,7 @@ class Bootstrap:
         return dict(
             bootstrap_type=self.bootstrap_type,
             controller='yes' if self.controller else '',
-            python_versions=self.python_versions,
+            python_interpreters=[f'{key}:{value}' for key, value in self.python_interpreters.items()],
             ssh_key_type=self.ssh_key.KEY_TYPE,
             ssh_private_key=self.ssh_key.key_contents,
             ssh_public_key=self.ssh_key.pub_contents,

test/lib/ansible_test/_internal/host_profiles.py

@@ -958,7 +958,7 @@ class DockerProfile(ControllerHostProfile[DockerConfig], SshTargetHostProfile[Do
         """Perform out-of-band setup before delegation."""
         bootstrapper = BootstrapDocker(
             controller=self.controller,
-            python_versions=[self.python.version],
+            python_interpreters={self.python.version: self.python.path},
             ssh_key=SshKey(self.args),
         )
 
@@ -1214,8 +1214,9 @@ class PosixRemoteProfile(ControllerHostProfile[PosixRemoteConfig], RemoteProfile
     def configure(self) -> None:
         """Perform in-band configuration. Executed before delegation for the controller and after delegation for targets."""
         # a target uses a single python version, but a controller may include additional versions for targets running on the controller
-        python_versions = [self.python.version] + [target.python.version for target in self.targets if isinstance(target, ControllerConfig)]
-        python_versions = sorted_versions(list(set(python_versions)))
+        python_interpreters = {self.python.version: self.python.path}
+        python_interpreters.update({target.python.version: target.python.path for target in self.targets if isinstance(target, ControllerConfig)})
+        python_interpreters = {version: python_interpreters[version] for version in sorted_versions(list(python_interpreters.keys()))}
 
         core_ci = self.wait_for_instance()
         pwd = self.wait_until_ready()
@@ -1226,7 +1227,7 @@ class PosixRemoteProfile(ControllerHostProfile[PosixRemoteConfig], RemoteProfile
             controller=self.controller,
             platform=self.config.platform,
             platform_version=self.config.version,
-            python_versions=python_versions,
+            python_interpreters=python_interpreters,
             ssh_key=core_ci.ssh_key,
         )
 

test/lib/ansible_test/_util/target/setup/bootstrap.sh

@@ -2,6 +2,16 @@
 
 set -eu
 
+remove_externally_managed_marker()
+{
+    "${python_interpreter}" -c '
+import pathlib
+import sysconfig
+path = pathlib.Path(sysconfig.get_path("stdlib")) / "EXTERNALLY-MANAGED"
+path.unlink(missing_ok=True)
+'
+}
+
 install_ssh_keys()
 {
     if [ ! -f "${ssh_private_key_path}" ]; then
@@ -340,14 +350,27 @@ bootstrap_docker()
 {
     # Required for newer mysql-server packages to install/upgrade on Ubuntu 16.04.
     rm -f /usr/sbin/policy-rc.d
+
+    for key_value in ${python_interpreters}; do
+        IFS=':' read -r python_version python_interpreter << EOF
+${key_value}
+EOF
+
+        echo "Bootstrapping Python ${python_version} at: ${python_interpreter}"
+
+        remove_externally_managed_marker
+    done
 }
 
 bootstrap_remote()
 {
-    for python_version in ${python_versions}; do
-        echo "Bootstrapping Python ${python_version}"
+    for key_value in ${python_interpreters}; do
+        IFS=':' read -r python_version python_interpreter << EOF
+${key_value}
+EOF
+
+        echo "Bootstrapping Python ${python_version} at: ${python_interpreter}"
 
-        python_interpreter="python${python_version}"
         python_package_version="$(echo "${python_version}" | tr -d '.')"
 
         case "${platform}" in
@@ -358,6 +381,8 @@ bootstrap_remote()
             "rhel") bootstrap_remote_rhel ;;
             "ubuntu") bootstrap_remote_ubuntu ;;
         esac
+
+        remove_externally_managed_marker
     done
 }
 
@@ -384,7 +409,7 @@ bootstrap_type=#{bootstrap_type}
 controller=#{controller}
 platform=#{platform}
 platform_version=#{platform_version}
-python_versions=#{python_versions}
+python_interpreters=#{python_interpreters}
 ssh_key_type=#{ssh_key_type}
 ssh_private_key=#{ssh_private_key}
 ssh_public_key=#{ssh_public_key}