import.py: use pathlib and support Windows

This commit is contained in:
linsui 2021-06-08 18:33:22 +08:00
parent 7ddcbb3e80
commit d6eece6395
6 changed files with 175 additions and 153 deletions

View File

@ -862,10 +862,8 @@ def write_status_json(output, pretty=False, name=None):
def get_head_commit_id(git_repo):
"""Get git commit ID for HEAD as a str
repo.head.commit.binsha is a bytearray stored in a str
"""
return hexlify(bytearray(git_repo.head.commit.binsha)).decode()
return git_repo.head.commit.hexsha
def setup_vcs(app):
@ -914,6 +912,8 @@ class vcs:
def __init__(self, remote, local):
# TODO: Remove this in Python3.6
local = str(local)
# svn, git-svn and bzr may require auth
self.username = None
if self.repotype() in ('git-svn', 'bzr'):
@ -1611,7 +1611,8 @@ def parse_androidmanifests(paths, app):
max_package = None
for path in paths:
# TODO: Remove this in Python3.6
path = str(path)
if not os.path.isfile(path):
continue
@ -1803,11 +1804,12 @@ def is_strict_application_id(name):
def get_all_gradle_and_manifests(build_dir):
paths = []
for root, dirs, files in os.walk(build_dir):
# TODO: Python3.6: Accepts a path-like object.
for root, dirs, files in os.walk(str(build_dir)):
for f in sorted(files):
if f == 'AndroidManifest.xml' \
or f.endswith('.gradle') or f.endswith('.gradle.kts'):
full = os.path.join(root, f)
full = Path(root) / f
paths.append(full)
return paths
@ -1817,22 +1819,18 @@ def get_gradle_subdir(build_dir, paths):
first_gradle_dir = None
for path in paths:
if not first_gradle_dir:
first_gradle_dir = os.path.relpath(os.path.dirname(path), build_dir)
if os.path.exists(path) and SETTINGS_GRADLE_REGEX.match(os.path.basename(path)):
with open(path) as fp:
for m in GRADLE_SUBPROJECT_REGEX.finditer(fp.read()):
for f in glob.glob(os.path.join(os.path.dirname(path), m.group(1), 'build.gradle*')):
with open(f) as fp:
while True:
line = fp.readline()
if not line:
break
if ANDROID_PLUGIN_REGEX.match(line):
return os.path.relpath(os.path.dirname(f), build_dir)
if first_gradle_dir and first_gradle_dir != '.':
first_gradle_dir = path.parent.relative_to(build_dir)
if path.exists() and SETTINGS_GRADLE_REGEX.match(str(path.name)):
for m in GRADLE_SUBPROJECT_REGEX.finditer(path.read_text()):
for f in (path.parent / m.group(1)).glob('build.gradle*'):
with f.open() as fp:
for line in fp.readlines():
if ANDROID_PLUGIN_REGEX.match(line):
return f.parent.relative_to(build_dir)
if first_gradle_dir and first_gradle_dir != Path('.'):
return first_gradle_dir
return ''
return
def getrepofrompage(url):
@ -4324,3 +4322,10 @@ NDKS = [
"url": "https://dl.google.com/android/repository/android-ndk-r22b-linux-x86_64.zip"
}
]
def handle_retree_error_on_windows(function, path, excinfo):
"""Python can't remove a readonly file on Windows so chmod first"""
if function in (os.unlink, os.rmdir, os.remove) and excinfo[0] == PermissionError:
os.chmod(path, stat.S_IWRITE)
function(path)

View File

@ -20,12 +20,12 @@
import configparser
import git
import json
import os
import shutil
import sys
import yaml
from argparse import ArgumentParser
import logging
from pathlib import Path, PurePosixPath
try:
from yaml import CSafeLoader as SafeLoader
@ -44,15 +44,15 @@ options = None
# WARNING! This cannot be imported as a Python module, so reuseable functions need to go into common.py!
def clone_to_tmp_dir(app):
tmp_dir = 'tmp'
if not os.path.isdir(tmp_dir):
logging.info(_("Creating temporary directory"))
os.makedirs(tmp_dir)
tmp_dir = os.path.join(tmp_dir, 'importer')
if os.path.exists(tmp_dir):
shutil.rmtree(tmp_dir)
def clone_to_tmp_dir(app):
tmp_dir = Path('tmp')
tmp_dir.mkdir(exist_ok=True)
tmp_dir = tmp_dir / 'importer'
if tmp_dir.exists():
shutil.rmtree(str(tmp_dir), onerror=common.handle_retree_error_on_windows)
vcs = common.getvcs(app.RepoType, app.Repo, tmp_dir)
vcs.gotorevision(options.rev)
@ -61,8 +61,8 @@ def clone_to_tmp_dir(app):
def check_for_kivy_buildozer(tmp_importer_dir, app, build):
versionCode = None
buildozer_spec = os.path.join(tmp_importer_dir, 'buildozer.spec')
if os.path.exists(buildozer_spec):
buildozer_spec = tmp_importer_dir / 'buildozer.spec'
if buildozer_spec.exists():
config = configparser.ConfigParser()
config.read(buildozer_spec)
import pprint
@ -132,15 +132,16 @@ def main():
raise FDroidException(_("This repo already has local metadata: %s") % local_metadata_files[0])
build = metadata.Build()
if options.url is None and os.path.isdir('.git'):
if options.url is None and Path('.git').is_dir():
app = metadata.App()
app.AutoName = os.path.basename(os.getcwd())
app.AutoName = Path.cwd().name
app.RepoType = 'git'
if os.path.exists('build.gradle') or os.path.exists('build.gradle.kts'):
if Path('build.gradle').exists() or Path('build.gradle.kts').exists():
build.gradle = ['yes']
git_repo = git.repo.Repo(os.getcwd())
# TODO: Python3.6: Should accept path-like
git_repo = git.Repo(str(Path.cwd()))
for remote in git.Remote.iter_items(git_repo):
if remote.name == 'origin':
url = git_repo.remotes.origin.url
@ -152,7 +153,9 @@ def main():
elif options.url:
app = common.get_app_from_url(options.url)
tmp_importer_dir = clone_to_tmp_dir(app)
git_repo = git.repo.Repo(tmp_importer_dir)
# TODO: Python3.6: Should accept path-like
git_repo = git.Repo(str(tmp_importer_dir))
if not options.omit_disable:
build.disable = 'Generated by import.py - check/set version fields and commit id'
write_local_file = False
@ -189,37 +192,36 @@ def main():
build.subdir = options.subdir
build.gradle = ['yes']
elif subdir:
build.subdir = subdir
build.subdir = str(PurePosixPath(subdir))
build.gradle = ['yes']
if options.license:
app.License = options.license
if options.categories:
app.Categories = options.categories.split(',')
if os.path.exists(os.path.join(subdir, 'jni')):
if (subdir / 'jni').exists():
build.buildjni = ['yes']
if os.path.exists(os.path.join(subdir, 'build.gradle')) \
or os.path.exists(os.path.join(subdir, 'build.gradle')):
if (subdir / 'build.gradle').exists() or (subdir / 'build.gradle').exists():
build.gradle = ['yes']
package_json = os.path.join(tmp_importer_dir, 'package.json') # react-native
pubspec_yaml = os.path.join(tmp_importer_dir, 'pubspec.yaml') # flutter
if os.path.exists(package_json):
package_json = tmp_importer_dir / 'package.json' # react-native
pubspec_yaml = tmp_importer_dir / 'pubspec.yaml' # flutter
if package_json.exists():
build.sudo = ['apt-get update || apt-get update', 'apt-get install -t stretch-backports npm', 'npm install -g react-native-cli']
build.init = ['npm install']
with open(package_json) as fp:
with package_json.open() as fp:
data = json.load(fp)
app.AutoName = data.get('name', app.AutoName)
app.License = data.get('license', app.License)
app.Description = data.get('description', app.Description)
app.WebSite = data.get('homepage', app.WebSite)
app_json = os.path.join(tmp_importer_dir, 'app.json')
if os.path.exists(app_json):
with open(app_json) as fp:
app_json = tmp_importer_dir / 'app.json'
if app_json.exists():
with app_json.open() as fp:
data = json.load(fp)
app.AutoName = data.get('name', app.AutoName)
if os.path.exists(pubspec_yaml):
with open(pubspec_yaml) as fp:
if pubspec_yaml.exists():
with pubspec_yaml.open() as fp:
data = yaml.load(fp, Loader=SafeLoader)
app.AutoName = data.get('name', app.AutoName)
app.License = data.get('license', app.License)
@ -232,8 +234,8 @@ def main():
'$$flutter$$/bin/flutter build apk',
]
git_modules = os.path.join(tmp_importer_dir, '.gitmodules')
if os.path.exists(git_modules):
git_modules = tmp_importer_dir / '.gitmodules'
if git_modules.exists():
build.submodules = True
metadata.post_metadata_parse(app)
@ -241,24 +243,25 @@ def main():
app['Builds'].append(build)
if write_local_file:
metadata.write_metadata('.fdroid.yml', app)
metadata.write_metadata(Path('.fdroid.yml'), app)
else:
# Keep the repo directory to save bandwidth...
if not os.path.exists('build'):
os.mkdir('build')
build_dir = os.path.join('build', appid)
if os.path.exists(build_dir):
Path('build').mkdir(exist_ok=True)
build_dir = Path('build') / appid
if build_dir.exists():
logging.warning(_('{path} already exists, ignoring import results!')
.format(path=build_dir))
sys.exit(1)
elif tmp_importer_dir is not None:
shutil.move(tmp_importer_dir, build_dir)
with open('build/.fdroidvcs-' + appid, 'w') as f:
f.write(app.RepoType + ' ' + app.Repo)
elif tmp_importer_dir:
# For Windows: Close the repo or a git.exe instance holds handles to repo
git_repo.close()
# TODO: Python3.9: Accepts a path-like object for both src and dst.
shutil.move(str(tmp_importer_dir), str(build_dir))
Path('build/.fdroidvcs-' + appid).write_text(app.RepoType + ' ' + app.Repo)
metadatapath = os.path.join('metadata', appid + '.yml')
metadatapath = Path('metadata') / (appid + '.yml')
metadata.write_metadata(metadatapath, app)
logging.info("Wrote " + metadatapath)
logging.info("Wrote " + str(metadatapath))
if __name__ == "__main__":

View File

@ -978,6 +978,8 @@ build_cont = re.compile(r'^[ \t]')
def write_metadata(metadatapath, app):
# TODO: Remove this
metadatapath = str(metadatapath)
if metadatapath.endswith('.yml'):
if importlib.util.find_spec('ruamel.yaml'):
with open(metadatapath, 'w') as mf:

View File

@ -22,6 +22,7 @@ import yaml
import gzip
from zipfile import ZipFile
from unittest import mock
from pathlib import Path
localmodule = os.path.realpath(
@ -1223,26 +1224,28 @@ class CommonTest(unittest.TestCase):
def test_get_all_gradle_and_manifests(self):
"""Test whether the function works with relative and absolute paths"""
a = fdroidserver.common.get_all_gradle_and_manifests(os.path.join('source-files', 'cn.wildfirechat.chat'))
a = fdroidserver.common.get_all_gradle_and_manifests(Path('source-files/cn.wildfirechat.chat'))
paths = [
os.path.join('source-files', 'cn.wildfirechat.chat', 'avenginekit', 'build.gradle'),
os.path.join('source-files', 'cn.wildfirechat.chat', 'build.gradle'),
os.path.join('source-files', 'cn.wildfirechat.chat', 'chat', 'build.gradle'),
os.path.join('source-files', 'cn.wildfirechat.chat', 'client', 'build.gradle'),
os.path.join('source-files', 'cn.wildfirechat.chat', 'client', 'src', 'main', 'AndroidManifest.xml'),
os.path.join('source-files', 'cn.wildfirechat.chat', 'emojilibrary', 'build.gradle'),
os.path.join('source-files', 'cn.wildfirechat.chat', 'gradle', 'build_libraries.gradle'),
os.path.join('source-files', 'cn.wildfirechat.chat', 'imagepicker', 'build.gradle'),
os.path.join('source-files', 'cn.wildfirechat.chat', 'mars-core-release', 'build.gradle'),
os.path.join('source-files', 'cn.wildfirechat.chat', 'push', 'build.gradle'),
os.path.join('source-files', 'cn.wildfirechat.chat', 'settings.gradle'),
'avenginekit/build.gradle',
'build.gradle',
'chat/build.gradle',
'client/build.gradle',
'client/src/main/AndroidManifest.xml',
'emojilibrary/build.gradle',
'gradle/build_libraries.gradle',
'imagepicker/build.gradle',
'mars-core-release/build.gradle',
'push/build.gradle',
'settings.gradle',
]
paths = [Path('source-files/cn.wildfirechat.chat') / path for path in paths]
self.assertEqual(sorted(paths), sorted(a))
abspath = os.path.join(self.basedir, 'source-files', 'realm')
abspath = Path(self.basedir) / 'source-files/realm'
p = fdroidserver.common.get_all_gradle_and_manifests(abspath)
self.assertEqual(1, len(p))
self.assertTrue(p[0].startswith(abspath))
# TODO: Pathon3.9: self.assertTrue(p[0].is_relative_to(abspath))
self.assertTrue(abspath in p[0].parents)
def test_get_gradle_subdir(self):
subdirs = {
@ -1251,17 +1254,12 @@ class CommonTest(unittest.TestCase):
'org.tasks': 'app',
'ut.ewh.audiometrytest': 'app',
}
for f in (
'cn.wildfirechat.chat',
'com.anpmech.launcher',
'org.tasks',
'ut.ewh.audiometrytest',
):
build_dir = os.path.join('source-files', f)
for k, v in subdirs.items():
build_dir = Path('source-files') / k
paths = fdroidserver.common.get_all_gradle_and_manifests(build_dir)
logging.info(paths)
subdir = fdroidserver.common.get_gradle_subdir(build_dir, paths)
self.assertEqual(subdirs[f], subdir)
self.assertEqual(v, str(subdir))
def test_parse_srclib_spec_good(self):
self.assertEqual(fdroidserver.common.parse_srclib_spec('osmand-external-skia@android/oreo'),

View File

@ -2,41 +2,41 @@
# http://www.drdobbs.com/testing/unit-testing-with-python/240165163
import inspect
import logging
import optparse
import os
import requests
import shutil
import sys
import tempfile
import unittest
from unittest import mock
from pathlib import Path
localmodule = os.path.realpath(
os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..')
)
print('localmodule: ' + localmodule)
if localmodule not in sys.path:
sys.path.insert(0, localmodule)
import fdroidserver.common
import fdroidserver.metadata
import requests
from testcommon import TmpCwd
# work around the syntax error from: import fdroidserver.import
import import_proxy
localmodule = Path(__file__).resolve().parent.parent
print('localmodule: ' + str(localmodule))
if localmodule not in sys.path:
sys.path.insert(0, str(localmodule))
import fdroidserver.common
import fdroidserver.metadata
class ImportTest(unittest.TestCase):
'''fdroid import'''
def setUp(self):
logging.basicConfig(level=logging.DEBUG)
self.basedir = os.path.join(localmodule, 'tests')
self.tmpdir = os.path.abspath(os.path.join(self.basedir, '..', '.testfiles'))
if not os.path.exists(self.tmpdir):
os.makedirs(self.tmpdir)
os.chdir(self.basedir)
self.basedir = localmodule / 'tests'
self.tmpdir = localmodule / '.testfiles'
self.tmpdir.mkdir(exist_ok=True)
# TODO: Python3.6: Accepts a path-like object.
os.chdir(str(self.basedir))
def test_import_gitlab(self):
# FDroidPopen needs some config to work
@ -57,59 +57,72 @@ class ImportTest(unittest.TestCase):
self.assertEqual(app.Repo, 'https://gitlab.com/fdroid/ci-test-app.git')
def test_get_app_from_url(self):
testdir = tempfile.mkdtemp(
prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir
)
os.chdir(testdir)
os.mkdir(os.path.join(testdir, 'tmp'))
tmp_importer = os.path.join(testdir, 'tmp', 'importer')
data = (
(
'cn.wildfirechat.chat',
'https://github.com/wildfirechat/android-chat',
'0.6.9',
'23',
),
(
'com.anpmech.launcher',
'https://github.com/KeikaiLauncher/KeikaiLauncher',
'Unknown',
None,
),
(
'ut.ewh.audiometrytest',
'https://github.com/ReeceStevens/ut_ewh_audiometer_2014',
'1.65',
'14',
),
)
for appid, url, vn, vc in data:
shutil.rmtree(tmp_importer, ignore_errors=True)
shutil.copytree(
os.path.join(self.basedir, 'source-files', appid), tmp_importer
# TODO: Pytohn3.6: The dir parameter now accepts a path-like object.
with tempfile.TemporaryDirectory(dir=str(self.tmpdir)) as testdir, TmpCwd(
testdir
):
testdir = Path(testdir)
(testdir / 'tmp').mkdir()
tmp_importer = testdir / 'tmp/importer'
data = (
(
'cn.wildfirechat.chat',
'https://github.com/wildfirechat/android-chat',
'0.6.9',
'23',
),
(
'com.anpmech.launcher',
'https://github.com/KeikaiLauncher/KeikaiLauncher',
'Unknown',
None,
),
(
'ut.ewh.audiometrytest',
'https://github.com/ReeceStevens/ut_ewh_audiometer_2014',
'1.65',
'14',
),
)
for appid, url, vn, vc in data:
# TODO: Python3.6: Accepts a path-like object.
shutil.rmtree(
str(tmp_importer),
onerror=fdroidserver.common.handle_retree_error_on_windows,
)
shutil.copytree(
str(self.basedir / 'source-files' / appid), str(tmp_importer)
)
app = fdroidserver.common.get_app_from_url(url)
with mock.patch('fdroidserver.common.getvcs',
lambda a, b, c: fdroidserver.common.vcs(url, testdir)):
with mock.patch('fdroidserver.common.vcs.gotorevision',
lambda s, rev: None):
with mock.patch('shutil.rmtree', lambda a: None):
build_dir = import_proxy.clone_to_tmp_dir(app)
self.assertEqual('git', app.RepoType)
self.assertEqual(url, app.Repo)
self.assertEqual(url, app.SourceCode)
logging.info(build_dir)
paths = fdroidserver.common.get_all_gradle_and_manifests(build_dir)
self.assertNotEqual(paths, [])
versionName, versionCode, package = fdroidserver.common.parse_androidmanifests(paths, app)
self.assertEqual(vn, versionName)
self.assertEqual(vc, versionCode)
self.assertEqual(appid, package)
app = fdroidserver.common.get_app_from_url(url)
with mock.patch(
'fdroidserver.common.getvcs',
lambda a, b, c: fdroidserver.common.vcs(url, testdir),
), mock.patch(
'fdroidserver.common.vcs.gotorevision', lambda s, rev: None
), mock.patch(
'shutil.rmtree', lambda a, onerror=None: None
):
build_dir = import_proxy.clone_to_tmp_dir(app)
self.assertEqual('git', app.RepoType)
self.assertEqual(url, app.Repo)
self.assertEqual(url, app.SourceCode)
logging.info(build_dir)
paths = fdroidserver.common.get_all_gradle_and_manifests(build_dir)
self.assertNotEqual(paths, [])
(
versionName,
versionCode,
package,
) = fdroidserver.common.parse_androidmanifests(paths, app)
self.assertEqual(vn, versionName)
self.assertEqual(vc, versionCode)
self.assertEqual(appid, package)
if __name__ == "__main__":
os.chdir(os.path.dirname(__file__))
# TODO: Python3.6: Support added to accept objects implementing the os.PathLike interface.
os.chdir(str(Path(__file__).parent))
parser = optparse.OptionParser()
parser.add_option(

View File

@ -1,20 +1,21 @@
# workaround the syntax error from: import fdroidserver.import
import inspect
import os
import sys
from pathlib import Path
localmodule = os.path.realpath(
os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..'))
print('localmodule: ' + localmodule)
localmodule = Path(__file__).resolve().parent.parent
print('localmodule: ' + str(localmodule))
if localmodule not in sys.path:
sys.path.insert(0, localmodule)
sys.path.insert(0, str(localmodule))
class Options:
def __init__(self):
self.rev = None
self.subdir = None
module = __import__('fdroidserver.import')
for name, obj in inspect.getmembers(module):
if name == 'import':