Merge branch 'extlib_scanignore' into 'master'

minor bugfixes

Closes #795, #796, and #759

See merge request fdroid/fdroidserver!771
This commit is contained in:
Marcus 2020-08-25 19:53:44 +00:00
commit e169238c60
8 changed files with 252 additions and 15 deletions

View File

@ -994,12 +994,14 @@ class vcs_git(vcs):
if p.returncode != 0:
lines = p.output.splitlines()
if 'Multiple remote HEAD branches' not in lines[0]:
raise VCSException(_("Git remote set-head failed"), p.output)
branch = lines[1].split(' ')[-1]
p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--', branch],
cwd=self.local, output=False)
if p2.returncode != 0:
raise VCSException(_("Git remote set-head failed"), p.output + '\n' + p2.output)
logging.warning(_("Git remote set-head failed: \"%s\"") % p.output.strip())
else:
branch = lines[1].split(' ')[-1]
p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--', branch],
cwd=self.local, output=False)
if p2.returncode != 0:
logging.warning(_("Git remote set-head failed: \"%s\"")
% p.output.strip() + '\n' + p2.output.strip())
self.refreshed = True
# origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
# a github repo. Most of the time this is the same as origin/master.
@ -2111,6 +2113,13 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
if not os.path.exists(libsrc):
raise BuildException("Missing extlib file {0}".format(libsrc))
shutil.copyfile(libsrc, os.path.join(libsdir, libf))
# Add extlibs to scanignore (this is relative to the build dir root, *sigh*)
if build.subdir:
scanignorepath = os.path.join(build.subdir, 'libs', libf)
else:
scanignorepath = os.path.join('libs', libf)
if scanignorepath not in build.scanignore:
build.scanignore.append(scanignorepath)
# Run a pre-build command if one is required
if build.prebuild:

View File

@ -134,7 +134,7 @@ def make_v1(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_
return sorted(list(obj))
if isinstance(obj, datetime):
# Java prefers milliseconds
# we also need to accound for time zone/daylight saving time
# we also need to account for time zone/daylight saving time
return int(calendar.timegm(obj.timetuple()) * 1000)
if isinstance(obj, dict):
d = collections.OrderedDict()

View File

@ -62,7 +62,7 @@ def scan_binary(apkfile):
usual_suspects = {
# The `apkanalyzer dex packages` output looks like this:
# M d 1 1 93 <packagename> <other stuff>
# The first column has P/C/M/F for package, class, methos or field
# The first column has P/C/M/F for package, class, method or field
# The second column has x/k/r/d for removed, kept, referenced and defined.
# We already filter for defined only in the apkanalyzer call. 'r' will be
# for things referenced but not distributed in the apk.
@ -177,6 +177,11 @@ def scan_source(build_dir, build=metadata.Build()):
return False
def ignoreproblem(what, path_in_build_dir):
"""
:param what: string describing the problem, will be printed in log messages
:param path_in_build_dir: path to the file relative to `build`-dir
"returns: 0 as we explicitly ignore the file, so don't count an error
"""
msg = ('Ignoring %s at %s' % (what, path_in_build_dir))
logging.info(msg)
if json_per_build is not None:
@ -184,14 +189,31 @@ def scan_source(build_dir, build=metadata.Build()):
return 0
def removeproblem(what, path_in_build_dir, filepath):
"""
:param what: string describing the problem, will be printed in log messages
:param path_in_build_dir: path to the file relative to `build`-dir
:param filepath: Path (relative to our current path) to the file
"returns: 0 as we deleted the offending file
"""
msg = ('Removing %s at %s' % (what, path_in_build_dir))
logging.info(msg)
if json_per_build is not None:
json_per_build['infos'].append([msg, path_in_build_dir])
os.remove(filepath)
try:
os.remove(filepath)
except FileNotFoundError:
# File is already gone, nothing to do.
# This can happen if we find multiple problems in one file that is setup for scandelete
# I.e. build.gradle files containig multiple unknown maven repos.
pass
return 0
def warnproblem(what, path_in_build_dir):
"""
:param what: string describing the problem, will be printed in log messages
:param path_in_build_dir: path to the file relative to `build`-dir
:returns: 0, as warnings don't count as errors
"""
if toignore(path_in_build_dir):
return 0
logging.warning('Found %s at %s' % (what, path_in_build_dir))
@ -200,6 +222,14 @@ def scan_source(build_dir, build=metadata.Build()):
return 0
def handleproblem(what, path_in_build_dir, filepath):
"""Dispatches to problem handlers (ignore, delete, warn) or returns 1
for increasing the error count
:param what: string describing the problem, will be printed in log messages
:param path_in_build_dir: path to the file relative to `build`-dir
:param filepath: Path (relative to our current path) to the file
:returns: 0 if the problem was ignored/deleted/is only a warning, 1 otherwise
"""
if toignore(path_in_build_dir):
return ignoreproblem(what, path_in_build_dir)
if todelete(path_in_build_dir):

View File

@ -1974,12 +1974,6 @@ def apply_info_from_latest_apk(apps, apks):
if app.NoSourceSince:
apk['antiFeatures'].add('NoSourceSince')
if 'added' in apk:
if not app.added or apk['added'] < app.added:
app.added = apk['added']
if not app.lastUpdated or apk['added'] > app.lastUpdated:
app.lastUpdated = apk['added']
if not app.added:
logging.debug("Don't know when " + appid + " was added")
if not app.lastUpdated:
@ -2166,6 +2160,27 @@ def create_metadata_from_template(apk):
logging.info(_("Generated skeleton metadata for {appid}").format(appid=apk['packageName']))
def read_added_date_from_all_apks(apps, apks):
"""
Added dates come from the stats/known_apks.txt file but are
read when scanning apks and thus need to be applied form apk
level to app level for _all_ apps and not only from non-archived
ones
TODO: read the added dates directly from known_apks.txt instead of
going through apks that way it also works for for repos that
don't keep an archive of apks.
"""
for appid, app in apps.items():
for apk in apks:
if apk['packageName'] == appid:
if 'added' in apk:
if not app.added or apk['added'] < app.added:
app.added = apk['added']
if not app.lastUpdated or apk['added'] > app.lastUpdated:
app.lastUpdated = apk['added']
def read_names_from_apks(apps, apks):
"""This is a stripped down copy of apply_info_from_latest_apk that only parses app names"""
for appid, app in apps.items():
@ -2384,6 +2399,10 @@ def main():
# This will be done again (as part of apply_info_from_latest_apk) for repo and archive
# separately later on, but it's fairly cheap anyway.
read_names_from_apks(apps, apks + archapks)
# The added date currently comes from the oldest apk which might be in the archive.
# So we need this populated at app level before continuing with only processing /repo
# or /archive
read_added_date_from_all_apks(apps, apks + archapks)
if len(repodirs) > 1:
archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])

View File

@ -23,6 +23,7 @@ if localmodule not in sys.path:
import fdroidserver.build
import fdroidserver.common
import fdroidserver.metadata
import fdroidserver.scanner
class BuildTest(unittest.TestCase):
@ -198,6 +199,50 @@ class BuildTest(unittest.TestCase):
self.assertFalse(os.path.exists('gen'))
self.assertFalse(os.path.exists('gradle-wrapper.jar'))
def test_scan_with_extlib(self):
testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
os.chdir(testdir)
os.mkdir("build")
config = dict()
config['sdk_path'] = os.getenv('ANDROID_HOME')
config['ndk_paths'] = {'r10d': os.getenv('ANDROID_NDK_HOME')}
config['build_tools'] = 'FAKE_BUILD_TOOLS_VERSION'
fdroidserver.common.config = config
app = fdroidserver.metadata.App()
app.id = 'com.gpl.rpg.AndorsTrail'
build = fdroidserver.metadata.Build()
build.commit = 'master'
build.androidupdate = ['no']
os.makedirs("extlib/android")
# write a fake binary jar file the scanner should definitely error on
with open('extlib/android/android-support-v4r11.jar', 'wb') as file:
file.write(b'PK\x03\x04\x14\x00\x08\x00\x08\x00-\x0eiA\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\t\x00\x04\x00META-INF/\xfe\xca\x00\x00\x03\x00PK\x07\x08\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00')
class FakeVcs():
# no need to change to the correct commit here
def gotorevision(self, rev, refresh=True):
pass
def getsrclib(self):
return None
# Test we trigger a scanner error without extlibs
build.extlibs = []
os.makedirs('build/libs')
shutil.copy('extlib/android/android-support-v4r11.jar', 'build/libs')
fdroidserver.common.prepare_source(FakeVcs(), app, build,
"build", "ignore", "extlib")
count = fdroidserver.scanner.scan_source("build", build)
self.assertEqual(1, count, "Should produce a scanner error without extlib")
# Now try again as an extlib
build.extlibs = ['android/android-support-v4r11.jar']
fdroidserver.common.prepare_source(FakeVcs(), app, build,
"build", "ignore", "extlib")
count = fdroidserver.scanner.scan_source("build", build)
self.assertEqual(0, count, "Shouldn't error on jar from extlib")
if __name__ == "__main__":
os.chdir(os.path.dirname(__file__))

View File

@ -607,6 +607,29 @@ grep -F 'info.guardianproject.urzip_100.apk' repo/index-v1.json repo/index.xml
grep -F 'info.guardianproject.urzip_100_b4964fd.apk' repo/index-v1.json
! grep -F 'info.guardianproject.urzip_100_b4964fd.apk' repo/index.xml
#------------------------------------------------------------------------------#
echo_header "test for added date being set correctly for repo and archive"
REPOROOT=`create_test_dir`
cd $REPOROOT
fdroid_init_with_prebuilt_keystore
mkdir -p {repo,archive,metadata,stats}
cp $WORKSPACE/tests/repo/com.politedroid_5.apk archive
cp $WORKSPACE/tests/repo/com.politedroid_6.apk repo
cp $WORKSPACE/tests/metadata/com.politedroid.yml metadata
#TODO: the timestamp of the oldest apk in the file should be used, even if that
# doesn't exist anymore
echo "com.politedroid_4.apk com.politedroid 2016-01-01" > stats/known_apks.txt
echo "com.politedroid_5.apk com.politedroid 2017-01-01" >> stats/known_apks.txt
echo "com.politedroid_6.apk com.politedroid 2018-01-01" >> stats/known_apks.txt
sed -i -e 's/ArchivePolicy:.*/ArchivePolicy: 1 versions/' metadata/com.politedroid.yml
# Get the java ms timestamp from UTC time
timestamp=$(date -u --date=2017-01-01 +%s)000
$fdroid update --pretty --nosign
grep -F "\"added\": $timestamp" repo/index-v1.json
# the archive will have the added timestamp for the app and for the apk, both need to be there
if [ $(grep -F "\"added\": $timestamp" archive/index-v1.json | wc -l) == 2 ]; then true; else false;fi
#------------------------------------------------------------------------------#
echo_header "test whatsnew from fastlane without CVC set"
REPOROOT=`create_test_dir`

View File

@ -270,6 +270,27 @@ class ScannerTest(unittest.TestCase):
self.assertTrue(found, 'this block should produce a URL:\n' + entry)
self.assertEqual(len(data), len(urls), 'each data example should produce a URL')
def test_scan_gradle_file_with_multiple_problems(self):
"""Check that the scanner can handle scandelete with gradle files with multiple problems"""
testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
os.chdir(testdir)
fdroidserver.scanner.config = None
fdroidserver.scanner.options = mock.Mock()
build = fdroidserver.metadata.Build()
build.scandelete = ['build.gradle']
with open('build.gradle', 'w') as fp:
fp.write(textwrap.dedent("""
maven {
url 'https://maven.fabric.io/public'
}
maven {
url 'https://evilcorp.com/maven'
}
"""))
count = fdroidserver.scanner.scan_source(testdir, build)
self.assertFalse(os.path.exists("build.gradle"))
self.assertEqual(0, count, 'there should be this many errors')
if __name__ == "__main__":
os.chdir(os.path.dirname(__file__))

90
tests/vcs.TestCase Executable file
View File

@ -0,0 +1,90 @@
#!/usr/bin/env python3
# http://www.drdobbs.com/testing/unit-testing-with-python/240165163
import inspect
import logging
import optparse
import os
import sys
import tempfile
import unittest
from git import Repo
localmodule = os.path.realpath(
os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..'))
print('localmodule: ' + localmodule)
if localmodule not in sys.path:
sys.path.insert(0, localmodule)
import fdroidserver.build
import fdroidserver.common
import fdroidserver.metadata
import fdroidserver.scanner
class VCSTest(unittest.TestCase):
"""For some reason the VCS classes are in fdroidserver/common.py"""
def setUp(self):
logging.basicConfig(level=logging.DEBUG)
self.basedir = os.path.join(localmodule, 'tests')
self.tmpdir = os.path.abspath(os.path.join(self.basedir, '..', '.testfiles'))
if not os.path.exists(self.tmpdir):
os.makedirs(self.tmpdir)
os.chdir(self.basedir)
def test_remote_set_head_can_fail(self):
testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
os.chdir(testdir)
# First create an upstream repo with one commit
upstream_repo = Repo.init("upstream_repo")
with open(upstream_repo.working_dir + "/file", 'w') as f:
f.write("Hello World!")
upstream_repo.index.add([upstream_repo.working_dir + "/file"])
upstream_repo.index.commit("initial commit")
commitid = upstream_repo.head.commit.hexsha
# Now clone it once manually, like gitlab runner gitlab-runner sets up a repo during CI
clone1 = Repo.init("clone1")
clone1.create_remote("upstream", "file://" + upstream_repo.working_dir)
clone1.remote("upstream").fetch()
clone1.head.reference = clone1.commit(commitid)
clone1.head.reset(index=True, working_tree=True)
self.assertTrue(clone1.head.is_detached)
# and now we want to use this clone as a source repo for fdroid build
config = {}
os.mkdir("build")
config['sdk_path'] = os.getenv('ANDROID_HOME')
config['ndk_paths'] = {'r10d': os.getenv('ANDROID_NDK_HOME')}
config['build_tools'] = 'FAKE_BUILD_TOOLS_VERSION'
config['java_paths'] = {'fake': 'fake'}
fdroidserver.common.config = config
app = fdroidserver.metadata.App()
app.RepoType = 'git'
app.Repo = clone1.working_dir
app.id = 'com.gpl.rpg.AndorsTrail'
build = fdroidserver.metadata.Build()
build.commit = commitid
build.androidupdate = ['no']
vcs, build_dir = fdroidserver.common.setup_vcs(app)
# force an init of the repo, the remote head error only occurs on the second gotorevision call
vcs.gotorevision(build.commit)
fdroidserver.common.prepare_source(vcs, app, build,
build_dir=build_dir, srclib_dir="ignore", extlib_dir="ignore")
self.assertTrue(os.path.isfile("build/com.gpl.rpg.AndorsTrail/file"))
if __name__ == "__main__":
os.chdir(os.path.dirname(__file__))
parser = optparse.OptionParser()
parser.add_option("-v", "--verbose", action="store_true", default=False,
help="Spew out even more information than normal")
(fdroidserver.common.options, args) = parser.parse_args(['--verbose'])
newSuite = unittest.TestSuite()
newSuite.addTest(unittest.makeSuite(VCSTest))
unittest.main(failfast=False)