opensourcemirror
/
fdroid-server
mirror of https://gitlab.com/fdroid/fdroidserver.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'even_more_docstrings' into 'master' 

Add more docstrings, replace pydocstyle with ruff

See merge request fdroid/fdroidserver!1432
This commit is contained in:
FestplattenSchnitzel 2024-04-12 15:46:47 +00:00
parent e24640da85 ea8c51ab12
commit 4c673a5ee7
11 changed files with 392 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored
View File

@ -78,3 +78,7 @@ locale/*/LC_MESSAGES/fdroidserver.mo
# sphinx
public/
/docs/build/
/docs/source/fdroidserver*.rst
/docs/source/modules.rst

15
.gitlab-ci.yml
View File

@ -284,6 +284,17 @@ black:
- apt-get install black
- black --check --diff --color $CI_PROJECT_DIR
# Replaces pydocstyle
ruff-check:
image: debian:bookworm-slim
<<: *apt-template
script:
- apt-get install python3-pip
- pip install --break-system-packages ruff
- ruff check fdroidserver --extend-exclude 'apksigcopier.py,common.py,deploy.py,exception.py,gpgsign.py,index.py,__init__.py,init.py,install.py,lint.py,__main__.py,metadata.py,mirror.py,net.py,publish.py,readmeta.py,rewritemeta.py,scanner.py,signatures.py,signindex.py,tail.py,update.py,verify.py,vmtools.py'
fedora_latest:
image: fedora:latest
only:
@ -560,10 +571,8 @@ Build documentation:
image: debian:bookworm-slim
<<: *apt-template
script:
- apt-get install make python3-sphinx python3-numpydoc python3-pydata-sphinx-theme pydocstyle fdroidserver
- apt-get install make python3-sphinx python3-numpydoc python3-pydata-sphinx-theme fdroidserver
- apt purge fdroidserver
# ignore vendored files
- pydocstyle --verbose --match='(?!apksigcopier|looseversion|setup|test_).*\.py' fdroidserver
- cd docs
- sphinx-apidoc -o ./source ../fdroidserver -M -e
- PYTHONPATH=.. sphinx-autogen -o generated source/*.rst

34
fdroidserver/btlog.py
View File

@ -1,4 +1,5 @@
#!/usr/bin/env python3
"""Update the binary transparency log for a URL."""
#
# btlog.py - part of the FDroid server tools
# Copyright (C) 2017, Hans-Christoph Steiner <hans@eds.org>
@ -39,6 +40,7 @@ import shutil
import tempfile
import zipfile
from argparse import ArgumentParser
from typing import Optional
from . import _
from . import common
@ -50,14 +52,30 @@ options = None
def make_binary_transparency_log(
repodirs, btrepo='binary_transparency', url=None, commit_title='fdroid update'
repodirs: collections.abc.Iterable,
btrepo: str = 'binary_transparency',
url: Optional[str] = None,
commit_title: str = 'fdroid update',
):
"""Log the indexes in a standalone git repo to serve as a "binary transparency" log.
References
Parameters
----------
https://www.eff.org/deeplinks/2014/02/open-letter-to-tech-companies
repodirs
The directories of the F-Droid repository to generate the binary
transparency log for.
btrepo
The path to the Git repository of the binary transparency log.
url
The URL of the F-Droid repository to generate the binary transparency
log for.
commit_title
The commit title for commits in the binary transparency log Git
repository.
Notes
-----
Also see https://www.eff.org/deeplinks/2014/02/open-letter-to-tech-companies .
"""
logging.info('Committing indexes to ' + btrepo)
if os.path.exists(os.path.join(btrepo, '.git')):
@ -149,6 +167,16 @@ For more info on this idea:
def main():
"""Generate or update a binary transparency log for a F-Droid repository.
The behaviour of this function is influenced by the configuration file as
well as command line parameters.
Raises
------
:exc:`~fdroidserver.exception.FDroidException`
If the specified or default Git repository does not exist.
"""
global options
parser = ArgumentParser()

246
fdroidserver/build.py
View File

@ -1,4 +1,5 @@
#!/usr/bin/env python3
"""Build a package from source."""
#
# build.py - part of the FDroid server tools
# Copyright (C) 2010-2014, Ciaran Gultnieks, ciaran@ciarang.com
@ -31,6 +32,7 @@ import requests
import tempfile
import argparse
import logging
from collections.abc import Iterable, Callable
from gettext import ngettext
from pathlib import Path
@ -53,21 +55,44 @@ ssh_channel = None
# Note that 'force' here also implies test mode.
def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
def build_server(
app: metadata.App,
build: metadata.Build,
vcs: common.vcs,
build_dir: Path,
output_dir: str,
log_dir: str,
force: bool,
):
"""Do a build on the builder vm.
Parameters
----------
app
app metadata dict
The metadata of the app to build.
build
The build of the app to build.
vcs
version control system controller object
The version control system controller object of the app.
build_dir
local source-code checkout of app
The local source-code checkout directory of the app.
output_dir
target folder for the build result
The target folder for the build result.
log_dir
The directory in the VM where the build logs are getting stored.
force
Don't refresh the already cloned repository and make the build stop on
exceptions.
Raises
------
:exc:`~fdroidserver.exception.BuildException`
If Paramiko is not installed, a srclib directory or srclib metadata
file is unexpectedly missing, the build process in the VM failed or
output files of the build process are missing.
:exc:`~fdroidserver.exception.FDroidException`
If the Buildserver ID could not be obtained or copying a directory to
the server failed.
"""
global buildserverid, ssh_channel
@ -115,8 +140,8 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
# Put all the necessary files in place...
ftp.chdir(homedir)
# Helper to copy the contents of a directory to the server...
def send_dir(path):
def send_dir(path: str):
"""Copy the contents of a directory to the server."""
logging.debug("rsyncing %s to %s" % (path, ftp.getcwd()))
# TODO this should move to `vagrant rsync` from >= v1.5
try:
@ -315,7 +340,16 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
'no output present')
def force_gradle_build_tools(build_dir, build_tools):
def force_gradle_build_tools(build_dir: str, build_tools: str):
"""Manipulate build tools version used in top level gradle file.
Parameters
----------
build_dir
The directory to start looking for gradle files.
build_tools
The build tools version that should be forced to use.
"""
for root, dirs, files in os.walk(build_dir):
for filename in files:
if not filename.endswith('.gradle'):
@ -329,7 +363,7 @@ def force_gradle_build_tools(build_dir, build_tools):
path)
def transform_first_char(string, method):
def transform_first_char(string: str, method: Callable[[str], str]) -> str:
"""Use method() on the first character of string."""
if len(string) == 0:
return string
@ -338,10 +372,37 @@ def transform_first_char(string, method):
return method(string[0]) + string[1:]
def get_metadata_from_apk(app, build, apkfile):
def get_metadata_from_apk(
app: metadata.App, build: metadata.Build, apkfile: str
) -> tuple[int, str]:
"""Get the required metadata from the built APK.
VersionName is allowed to be a blank string, i.e. ''
Parameters
----------
app
The app metadata used to build the APK.
build
The build that resulted in the APK.
apkfile
The path of the APK file.
Returns
-------
versionCode
The versionCode from the APK or from the metadata is build.novcheck is
set.
versionName
The versionName from the APK or from the metadata is build.novcheck is
set.
Raises
------
:exc:`~fdroidserver.exception.BuildException`
If native code should have been built but was not packaged, no version
information or no package ID could be found or there is a mismatch
between the package ID in the metadata and the one found in the APK.
"""
appid, versionCode, versionName = common.get_apk_id(apkfile)
native_code = common.get_native_code(apkfile)
@ -361,8 +422,70 @@ def get_metadata_from_apk(app, build, apkfile):
return versionCode, versionName
def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver, refresh):
"""Do a build locally."""
def build_local(
app: metadata.App,
build: metadata.Build,
vcs: common.vcs,
build_dir: Path,
output_dir: str,
log_dir: str,
srclib_dir: str,
extlib_dir: str,
tmp_dir: str,
force: bool,
onserver: bool,
refresh: bool,
):
"""Do a build locally.
Parameters
----------
app
The metadata of the app to build.
build
The build of the app to build.
vcs
The version control system controller object of the app.
build_dir
The local source-code checkout directory of the app.
output_dir
The target folder for the build result.
log_dir
The directory in the VM where the build logs are getting stored.
srclib_dir
The path to the srclibs directory, usually 'build/srclib'.
extlib_dir
The path to the extlibs directory, usually 'build/extlib'.
tmp_dir
The temporary directory for building the source tarball.
force
Don't refresh the already cloned repository and make the build stop on
exceptions.
onserver
Assume the build is happening inside the VM.
refresh
Enable fetching the latest refs from the VCS remote.
Raises
------
:exc:`~fdroidserver.exception.BuildException`
If running a `sudo` command failed, locking the root account failed,
`sudo` couldn't be removed, cleaning the build environment failed,
skipping the scanning has been requested but `scandelete` is present,
errors occurred during scanning, running the `build` commands from the
metadata failed, building native code failed, building with the
specified build method failed, no output could be found with build
method `maven`, more or less than one APK were found with build method
`gradle`, less or more than one APKs match the `output` glob specified
in the metadata, running a `postbuild` command specified in the
metadata failed, the built APK is debuggable, the unsigned APK is not
at the expected location, the APK does not contain the expected
`versionName` and `versionCode` or undesired package names have been
found in the APK.
:exc:`~fdroidserver.exception.FDroidException`
If no Android NDK version could be found and the build isn't run in a
builder VM, the selected Android NDK is not a directory.
"""
ndk_path = build.ndk_path()
if build.ndk or (build.buildjni and build.buildjni != ['no']):
if not ndk_path:
@ -467,11 +590,11 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext
for root, dirs, files in os.walk(build_dir):
def del_dirs(dl):
def del_dirs(dl: Iterable):
for d in dl:
shutil.rmtree(os.path.join(root, d), ignore_errors=True)
def del_files(fl):
def del_files(fl: Iterable):
for f in fl:
if f in files:
os.remove(os.path.join(root, f))
@ -759,30 +882,69 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext
os.path.join(output_dir, tarname))
def trybuild(app, build, build_dir, output_dir, log_dir, also_check_dir,
srclib_dir, extlib_dir, tmp_dir, repo_dir, vcs, test,
server, force, onserver, refresh):
def trybuild(
app: metadata.App,
build: metadata.Build,
build_dir: Path,
output_dir: str,
log_dir: str,
also_check_dir: str,
srclib_dir: str,
extlib_dir: str,
tmp_dir: str,
repo_dir: str,
vcs: common.vcs,
test: bool,
server: bool,
force: bool,
onserver: bool,
refresh: bool,
) -> bool:
"""Build a particular version of an application, if it needs building.
Parameters
----------
app
The metadata of the app to build.
build
The build of the app to build.
build_dir
The local source-code checkout directory of the app.
output_dir
The directory where the build output will go.
Usually this is the 'unsigned' directory.
The directory where the build output will go. Usually this is the
'unsigned' directory.
log_dir
The directory in the VM where the build logs are getting stored.
also_check_dir
An additional location for checking if the build is necessary (usually
the archive repo).
srclib_dir
The path to the srclibs directory, usually 'build/srclib'.
extlib_dir
The path to the extlibs directory, usually 'build/extlib'.
tmp_dir
The temporary directory for building the source tarball of the app to
build.
repo_dir
The repo directory - used for checking if the build is necessary.
also_check_dir
An additional location for checking if the build
is necessary (usually the archive repo)
vcs
The version control system controller object of the app to build.
test
True if building in test mode, in which case the build will
always happen, even if the output already exists. In test mode, the
output directory should be a temporary location, not any of the real
ones.
True if building in test mode, in which case the build will always
happen, even if the output already exists. In test mode, the output
directory should be a temporary location, not any of the real ones.
server
Use buildserver VM for building.
force
Build app regardless of disabled state or scanner errors.
onserver
Assume the build is happening inside the VM.
refresh
Enable fetching the latest refs from the VCS remote.
Returns
-------
Boolean
status
True if the build was done, False if it wasn't necessary.
"""
dest_file = common.get_release_filename(app, build)
@ -821,7 +983,13 @@ def trybuild(app, build, build_dir, output_dir, log_dir, also_check_dir,
def force_halt_build(timeout):
"""Halt the currently running Vagrant VM, to be called from a Timer."""
"""Halt the currently running Vagrant VM, to be called from a Timer.
Parameters
----------
timeout
The timeout in seconds.
"""
logging.error(_('Force halting build after {0} sec timeout!').format(timeout))
timeout_event.set()
if ssh_channel:
@ -830,7 +998,7 @@ def force_halt_build(timeout):
vm.destroy()
def keep_when_not_allowed():
def keep_when_not_allowed() -> bool:
"""Control if APKs signed by keys not in AllowedAPKSigningKeys are removed."""
return (
(options is not None and options.keep_when_not_allowed)
@ -839,13 +1007,15 @@ def keep_when_not_allowed():
)
def parse_commandline():
def parse_commandline() -> tuple[argparse.Namespace, argparse.ArgumentParser]:
"""Parse the command line.
Returns
-------
options
The resulting options parsed from the command line arguments.
parser
The argument parser.
"""
parser = argparse.ArgumentParser(usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]")
common.setup_global_opts(parser)
@ -905,6 +1075,22 @@ timeout_event = threading.Event()
def main():
"""Build a package from source.
The behaviour of this function is influenced by the configuration file as
well as command line parameters.
Raises
------
:exc:`~fdroidserver.exception.FDroidException`
If more than one local metadata file has been found, no app metadata
has been found, there are no apps to process, downloading binaries for
checking the reproducibility of a built binary failed, the built binary
is different from supplied reference binary, the reference binary is
signed with a different signing key than expected, a VCS error occured
while building an app or a different error occured while building an
app.
"""
global options, config, buildserverid, fdroidserverid
options, parser = parse_commandline()

1
fdroidserver/checkupdates.py
View File

@ -1,4 +1,5 @@
#!/usr/bin/env python3
"""Check for updates to applications."""
#
# checkupdates.py - part of the FDroid server tools
# Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com

60
fdroidserver/import_subcommand.py
View File

@ -1,4 +1,5 @@
#!/usr/bin/env python3
"""Extract application metadata from a source repository."""
#
# import_subcommand.py - part of the FDroid server tools
# Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
@ -30,6 +31,7 @@ import yaml
from argparse import ArgumentParser
import logging
from pathlib import Path
from typing import Optional
try:
from yaml import CSafeLoader as SafeLoader
@ -53,7 +55,19 @@ def handle_retree_error_on_windows(function, path, excinfo):
function(path)
def clone_to_tmp_dir(app):
def clone_to_tmp_dir(app: metadata.App) -> Path:
"""Clone the source repository of an app to a temporary directory for further processing.
Parameters
----------
app
The App instance to clone the source of.
Returns
-------
tmp_dir
The (temporary) directory the apps source has been cloned into.
"""
tmp_dir = Path('tmp')
tmp_dir.mkdir(exist_ok=True)
@ -67,7 +81,7 @@ def clone_to_tmp_dir(app):
return tmp_dir
def getrepofrompage(url):
def getrepofrompage(url: str) -> tuple[Optional[str], str]:
"""Get the repo type and address from the given web page.
The page is scanned in a rather naive manner for 'git clone xxxx',
@ -75,6 +89,17 @@ def getrepofrompage(url):
that's the information we want. Returns repotype, address, or
None, reason
Parameters
----------
url
The url to look for repository information at.
Returns
-------
repotype_or_none
The found repository type or None if an error occured.
address_or_reason
The address to the found repository or the reason if an error occured.
"""
if not url.startswith('http'):
return (None, _('{url} does not start with "http"!'.format(url=url)))
@ -120,13 +145,29 @@ def getrepofrompage(url):
return (None, _("No information found.") + page)
def get_app_from_url(url):
def get_app_from_url(url: str) -> metadata.App:
"""Guess basic app metadata from the URL.
The URL must include a network hostname, unless it is an lp:,
file:, or git/ssh URL. This throws ValueError on bad URLs to
match urlparse().
Parameters
----------
url
The URL to look to look for app metadata at.
Returns
-------
app
App instance with the found metadata.
Raises
------
:exc:`~fdroidserver.exception.FDroidException`
If the VCS type could not be determined.
:exc:`ValueError`
If the URL is invalid.
"""
parsed = urllib.parse.urlparse(url)
invalid_url = False
@ -183,6 +224,19 @@ def get_app_from_url(url):
def main():
"""Extract app metadata and write it to a file.
The behaviour of this function is influenced by the configuration file as
well as command line parameters.
Raises
------
:exc:`~fdroidserver.exception.FDroidException`
If the repository already has local metadata, no URL is specified and
the current directory is not a Git repository, no application ID could
be found, no Gradle project could be found or there is already metadata
for the found application ID.
"""
global config, options
# Parse command line...

61
fdroidserver/nightly.py
View File

@ -1,4 +1,5 @@
#!/usr/bin/env python3
"""Set up an app build for a nightly build repo."""
#
# nightly.py - part of the FDroid server tools
# Copyright (C) 2017 Hans-Christoph Steiner <hans@eds.org>
@ -32,6 +33,7 @@ import tempfile
import yaml
from urllib.parse import urlparse
from argparse import ArgumentParser
from typing import Optional
from . import _
from . import common
@ -48,12 +50,38 @@ DISTINGUISHED_NAME = 'CN=Android Debug,O=Android,C=US'
NIGHTLY = '-nightly'
def _get_keystore_secret_var(keystore):
def _get_keystore_secret_var(keystore: str) -> str:
"""Get keystore secret as base64.
Parameters
----------
keystore
The path of the keystore.
Returns
-------
base64_secret
The keystore secret as base64 string.
"""
with open(keystore, 'rb') as fp:
return base64.standard_b64encode(fp.read()).decode('ascii')
def _ssh_key_from_debug_keystore(keystore=None):
def _ssh_key_from_debug_keystore(keystore: Optional[str] = None) -> str:
"""Convert a debug keystore to an SSH private key.
This leaves the original keystore file in place.
Parameters
----------
keystore
The keystore to convert to a SSH private key.
Returns
-------
key_path
The SSH private key file path in the temporary directory.
"""
if keystore is None:
# set this here so it can be overridden in the tests
# TODO convert this to a class to get rid of this nonsense
@ -148,7 +176,23 @@ def _ssh_key_from_debug_keystore(keystore=None):
return ssh_private_key_file
def get_repo_base_url(clone_url, repo_git_base, force_type=None):
def get_repo_base_url(clone_url: str, repo_git_base: str, force_type: Optional[str] = None) -> str:
"""Generate the base URL for the F-Droid repository.
Parameters
----------
clone_url
The URL to clone the Git repository.
repo_git_base
The project path of the Git repository at the Git forge.
force_type
The Git forge of the project.
Returns
-------
repo_base_url
The base URL of the F-Droid repository.
"""
if force_type is None:
force_type = urlparse(clone_url).netloc
if force_type == 'gitlab.com':
@ -160,6 +204,17 @@ def get_repo_base_url(clone_url, repo_git_base, force_type=None):
def main():
"""Deploy to F-Droid repository or generate SSH private key from keystore.
The behaviour of this function is influenced by the configuration file as
well as command line parameters.
Raises
------
:exc:`~fdroidserver.exception.VCSException`
If the nightly Git repository could not be cloned during an attempt to
deploy.
"""
parser = ArgumentParser()
common.setup_global_opts(parser)
parser.add_argument(

6
hooks/pre-commit
View File

@ -80,7 +80,7 @@ find_command() {
}
DASH=$(find_command dash)
PYDOCSTYLE=$(find_command pydocstyle)
RUFF=$(find_command ruff)
PYFLAKES=$(find_command pyflakes)
PYCODESTYLE=$(find_command pycodestyle pep8)
RUBY=$(find_command ruby)
@ -91,8 +91,8 @@ if [ "$PY_FILES $PY_TEST_FILES" != " " ]; then
err "pyflakes tests failed!"
fi
# ignore vendored files
if ! $PYDOCSTYLE --match='(?!apksigcopier|looseversion).*\.py' $PY_FILES $PY_TEST_FILES; then
err "pydocstyle tests failed!"
if ! $RUFF check fdroidserver --extend-exclude 'apksigcopier.py,common.py,deploy.py,exception.py,gpgsign.py,index.py,__init__.py,init.py,install.py,lint.py,__main__.py,metadata.py,mirror.py,net.py,publish.py,readmeta.py,rewritemeta.py,scanner.py,signatures.py,signindex.py,tail.py,update.py,verify.py,vmtools.py'; then
err "ruff tests failed!"
fi
fi

10
pyproject.toml
View File

@ -142,3 +142,13 @@ max-nested-blocks = 5
[tool.pylint.format]
# Maximum number of characters on a single line.
max-line-length = 88
[tool.ruff]
extend-exclude = ["fdroidserver/apksigcopier.py", "fdroidserver/looseversion.py"]
[tool.ruff.lint]
select = ["D"]
[tool.ruff.lint.pydocstyle]
convention = "numpy"

9
setup.cfg
View File

@ -46,12 +46,3 @@ max-line-length = 88
[flake8]
ignore = E123,E203,E402,E501,W503
max-line-length = 88
# Settings for docstrings linter
# we use numpy stlye https://numpydoc.readthedocs.io/en/latest/format.html
# ignored errors are
# * D10*: Missing docstring *
# * rest are the conventions which are ignored by numpy conventions according to http://www.pydocstyle.org/en/stable/error_codes.html
[pydocstyle]
#convention = numpy # cannot be used in combination with ignore, so we list rules seperately.
ignore = D100,D101,D102,D103,D104,D105,D106,D107,D203,D212,D213,D402,D413,D415,D416,D417

1
setup.py
View File

@ -119,7 +119,6 @@ setup(
'sphinx',
'numpydoc',
'pydata_sphinx_theme',
'pydocstyle',
],
},
classifiers=[