Merge pull request #20856 from DeeDeeG/misc-python3-fixes

Miscellaneous python3-related updates and fixes

.gitignore

@@ -7,6 +7,7 @@ Thumbs.db
 .svn
 .nvm-version
 .vscode
+.python-version
 node_modules
 npm-debug.log
 debug.log

.python-version

@@ -1 +0,0 @@
-2.7.13

README.md

@@ -59,7 +59,8 @@ An archive is available for people who don't want to install `atom` as root.
 This version enables you to install multiple Atom versions in parallel. It has been built on Ubuntu 64-bit,
 but should be compatible with other Linux distributions.
 
-1. Install dependencies (on Ubuntu): `sudo apt install git gconf2 gconf-service libgtk2.0-0 libudev1 libgcrypt20 libnotify4 libxtst6 libnss3 python gvfs-bin xdg-utils libcap2`
+1. Install dependencies (on Ubuntu): `sudo apt install git gconf2 gconf-service libgtk2.0-0 libudev1 libgcrypt20 libnotify4 libxtst6 libnss3 python3 gvfs-bin xdg-utils libcap2`
+   1. (If the `python3` package isn't available, or is too old (Python 3 should be >= 3.5), either `python2` or `python` (2.6 or 2.7) will work in its place.)
 2. Download `atom-amd64.tar.gz` from the [Atom releases page](https://github.com/atom/atom/releases/latest).
 3. Run `tar xf atom-amd64.tar.gz` in the directory where you want to extract the Atom folder.
 4. Launch Atom using the installed `atom` command from the newly extracted directory.

resources/linux/debian/lintian-overrides

@@ -1,7 +0,0 @@
-atom: arch-dependent-file-in-usr-share
-atom: changelog-file-missing-in-native-package
-atom: copyright-file-contains-full-apache-2-license
-atom: copyright-should-refer-to-common-license-file-for-apache-2
-atom: embedded-library
-atom: package-installs-python-bytecode
-atom: unstripped-binary-or-object

script/lib/create-debian-package.js

@@ -54,11 +54,6 @@ module.exports = function(packagedAppPath) {
     debianPackageShareDirPath,
     'pixmaps'
   );
-  const debianPackageLintianOverridesDirPath = path.join(
-    debianPackageShareDirPath,
-    'lintian',
-    'overrides'
-  );
   const debianPackageDocsDirPath = path.join(
     debianPackageShareDirPath,
     'doc',
@@ -93,7 +88,6 @@ module.exports = function(packagedAppPath) {
   fs.mkdirpSync(debianPackageShareDirPath);
   fs.mkdirpSync(debianPackageApplicationsDirPath);
   fs.mkdirpSync(debianPackageIconsDirPath);
-  fs.mkdirpSync(debianPackageLintianOverridesDirPath);
   fs.mkdirpSync(debianPackageDocsDirPath);
   fs.mkdirpSync(debianPackageBinDirPath);
 
@@ -192,20 +186,6 @@ module.exports = function(packagedAppPath) {
     path.join(debianPackageDocsDirPath, 'copyright')
   );
 
-  console.log(
-    `Copying lintian overrides into "${debianPackageLintianOverridesDirPath}"`
-  );
-  fs.copySync(
-    path.join(
-      CONFIG.repositoryRootPath,
-      'resources',
-      'linux',
-      'debian',
-      'lintian-overrides'
-    ),
-    path.join(debianPackageLintianOverridesDirPath, atomExecutableName)
-  );
-
   console.log(
     `Copying polkit configuration into "${debianPackageShareDirPath}"`
   );

script/lib/verify-machine-requirements.js

@@ -1,7 +1,6 @@
 'use strict';
 
 const childProcess = require('child_process');
-const fs = require('fs');
 const path = require('path');
 
 const CONFIG = require('../config');
@@ -9,9 +8,7 @@ const CONFIG = require('../config');
 module.exports = function(ci) {
   verifyNode();
   verifyNpm(ci);
-  if (process.platform === 'win32') {
+  verifyPython();
-    verifyPython();
-  }
 };
 
 function verifyNode() {
@@ -50,36 +47,127 @@ function verifyNpm(ci) {
 }
 
 function verifyPython() {
-  const systemDrive = process.env.SystemDrive || 'C:\\';
+  // This function essentially re-implements node-gyp's "find-python.js" library,
-  let pythonExecutable;
+  // but in a synchronous, bootstrap-script-friendly way.
-  if (process.env.PYTHON) {
+  // It is based off of the logic of the file from node-gyp v5.x:
-    pythonExecutable = process.env.PYTHON;
+  // https://github.com/nodejs/node-gyp/blob/v5.1.1/lib/find-python.js
-  } else {
+  // This node-gyp is the version in use by current npm (in mid 2020).
-    const pythonBinPath = path.join(systemDrive, 'Python27', 'python.exe');
+  //
-    if (fs.existsSync(pythonBinPath)) {
+  // TODO: If this repo ships a newer version of node-gyp (v6.x or later), please update this script.
-      pythonExecutable = pythonBinPath;
+  // (Currently, the build scripts and apm each depend on npm v6.14, which depends on node-gyp v5.)
-    } else {
+  // Differences between major versions of node-gyp:
-      pythonExecutable = 'python';
+  // node-gyp 5.x looks for python, then python2, then python3.
+  // node-gyp 6.x looks for python3, then python, then python2.)
+  // node-gyp 5.x accepts Python ^2.6 || >= 3.5, node-gyp 6+ only accepts Python == 2.7 || >= 3.5.
+  // node-gyp 7.x stopped using the "-2" flag for "py.exe",
+  // so as to allow finding Python 3 as well, not just Python 2.
+  // https://github.com/nodejs/node-gyp/blob/master/CHANGELOG.md#v700-2020-06-03
+
+  let stdout;
+  let fullVersion;
+  let usablePythonWasFound;
+  let triedLog = '';
+  let binaryPlusFlag;
+
+  function verifyBinary(binary, prependFlag) {
+    if (binary && !usablePythonWasFound) {
+      // clear re-used "result" variables now that we're checking another python binary.
+      stdout = '';
+      fullVersion = '';
+
+      let allFlags = [
+        '-c',
+        'import platform\nprint(platform.python_version())'
+      ];
+      if (prependFlag) {
+        // prependFlag is an optional argument,
+        // used to prepend "-2" for the "py.exe" launcher.
+        //
+        // TODO: Refactor this script by eliminating "prependFlag"
+        // once we update to node-gyp v7.x or newer;
+        // the "-2" flag is not used in node-gyp v7.x.
+        allFlags.unshift(prependFlag);
+      }
+
+      try {
+        stdout = childProcess.execFileSync(binary, allFlags, {
+          env: process.env,
+          stdio: ['ignore', 'pipe', 'ignore']
+        });
+      } catch {}
+
+      if (stdout) {
+        if (stdout.indexOf('+') !== -1)
+          stdout = stdout.toString().replace(/\+/g, '');
+        if (stdout.indexOf('rc') !== -1)
+          stdout = stdout.toString().replace(/rc(.*)$/gi, '');
+        fullVersion = stdout.toString().trim();
+      }
+
+      if (fullVersion) {
+        let versionComponents = fullVersion.split('.');
+        let majorVersion = Number(versionComponents[0]);
+        let minorVersion = Number(versionComponents[1]);
+        if (
+          (majorVersion === 2 && minorVersion >= 6) ||
+          (majorVersion === 3 && minorVersion >= 5)
+        ) {
+          usablePythonWasFound = true;
+        }
+      }
+
+      // Prepare to log which commands were tried, and the results, in case no usable Python can be found.
+      if (prependFlag) {
+        binaryPlusFlag = binary + ' ' + prependFlag;
+      } else {
+        binaryPlusFlag = binary;
+      }
+      triedLog = triedLog.concat(
+        `log message: tried to check version of "${binaryPlusFlag}", got: "${fullVersion}"\n`
+      );
     }
   }
 
-  let stdout = childProcess.execFileSync(
+  function verifyForcedBinary(binary) {
-    pythonExecutable,
+    if (typeof binary !== 'undefined' && binary.length > 0) {
-    ['-c', 'import platform\nprint(platform.python_version())'],
+      verifyBinary(binary);
-    { env: process.env }
+      if (!usablePythonWasFound) {
-  );
+        throw new Error(
-  if (stdout.indexOf('+') !== -1) stdout = stdout.replace(/\+/g, '');
+          `NODE_GYP_FORCE_PYTHON is set to: "${binary}", but this is not a valid Python.\n` +
-  if (stdout.indexOf('rc') !== -1) stdout = stdout.replace(/rc(.*)$/gi, '');
+            'Please set NODE_GYP_FORCE_PYTHON to something valid, or unset it entirely.\n' +
-  const fullVersion = stdout.toString().trim();
+            '(Python 2.6, 2.7 or 3.5+ is required to build Atom.)\n'
-  const versionComponents = fullVersion.split('.');
+        );
-  const majorVersion = Number(versionComponents[0]);
+      }
-  const minorVersion = Number(versionComponents[1]);
+    }
-  if (majorVersion === 2 && minorVersion === 7) {
+  }
+
+  // These first two checks do nothing if the relevant
+  // environment variables aren't set.
+  verifyForcedBinary(process.env.NODE_GYP_FORCE_PYTHON);
+  // All the following checks will no-op if a previous check has succeeded.
+  verifyBinary(process.env.PYTHON);
+  verifyBinary('python');
+  verifyBinary('python2');
+  verifyBinary('python3');
+  if (process.platform === 'win32') {
+    verifyBinary('py.exe', '-2');
+    verifyBinary(
+      path.join(process.env.SystemDrive || 'C:', 'Python27', 'python.exe')
+    );
+    verifyBinary(
+      path.join(process.env.SystemDrive || 'C:', 'Python37', 'python.exe')
+    );
+  }
+
+  if (usablePythonWasFound) {
     console.log(`Python:\tv${fullVersion}`);
   } else {
     throw new Error(
-      `Python 2.7 is required to build Atom. ${pythonExecutable} returns version ${fullVersion}.\n` +
+      `\n${triedLog}\n` +
-        `Set the PYTHON env var to '/path/to/Python27/python.exe' if your python is installed in a non-default location.`
+        'Python 2.6, 2.7 or 3.5+ is required to build Atom.\n' +
+        'verify-machine-requirements.js was unable to find such a version of Python.\n' +
+        "Set the PYTHON env var to e.g. 'C:/path/to/Python27/python.exe'\n" +
+        'if your Python is installed in a non-default location.\n'
     );
   }
 }

script/vsts/platforms/windows.yml

@@ -22,7 +22,7 @@ jobs:
     steps:
       - task: UsePythonVersion@0
         inputs:
-          versionSpec: '2.7'
+          versionSpec: '3.8'
 
       - task: NodeTool@0
         inputs: