Merge branch 'extlib_scanignore' into 'master'
minor bugfixes Closes #795, #796, and #759 See merge request fdroid/fdroidserver!771
This commit is contained in:
commit
e169238c60
|
@ -994,12 +994,14 @@ class vcs_git(vcs):
|
|||
if p.returncode != 0:
|
||||
lines = p.output.splitlines()
|
||||
if 'Multiple remote HEAD branches' not in lines[0]:
|
||||
raise VCSException(_("Git remote set-head failed"), p.output)
|
||||
branch = lines[1].split(' ')[-1]
|
||||
p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--', branch],
|
||||
cwd=self.local, output=False)
|
||||
if p2.returncode != 0:
|
||||
raise VCSException(_("Git remote set-head failed"), p.output + '\n' + p2.output)
|
||||
logging.warning(_("Git remote set-head failed: \"%s\"") % p.output.strip())
|
||||
else:
|
||||
branch = lines[1].split(' ')[-1]
|
||||
p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--', branch],
|
||||
cwd=self.local, output=False)
|
||||
if p2.returncode != 0:
|
||||
logging.warning(_("Git remote set-head failed: \"%s\"")
|
||||
% p.output.strip() + '\n' + p2.output.strip())
|
||||
self.refreshed = True
|
||||
# origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
|
||||
# a github repo. Most of the time this is the same as origin/master.
|
||||
|
@ -2111,6 +2113,13 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
|
|||
if not os.path.exists(libsrc):
|
||||
raise BuildException("Missing extlib file {0}".format(libsrc))
|
||||
shutil.copyfile(libsrc, os.path.join(libsdir, libf))
|
||||
# Add extlibs to scanignore (this is relative to the build dir root, *sigh*)
|
||||
if build.subdir:
|
||||
scanignorepath = os.path.join(build.subdir, 'libs', libf)
|
||||
else:
|
||||
scanignorepath = os.path.join('libs', libf)
|
||||
if scanignorepath not in build.scanignore:
|
||||
build.scanignore.append(scanignorepath)
|
||||
|
||||
# Run a pre-build command if one is required
|
||||
if build.prebuild:
|
||||
|
|
|
@ -134,7 +134,7 @@ def make_v1(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_
|
|||
return sorted(list(obj))
|
||||
if isinstance(obj, datetime):
|
||||
# Java prefers milliseconds
|
||||
# we also need to accound for time zone/daylight saving time
|
||||
# we also need to account for time zone/daylight saving time
|
||||
return int(calendar.timegm(obj.timetuple()) * 1000)
|
||||
if isinstance(obj, dict):
|
||||
d = collections.OrderedDict()
|
||||
|
|
|
@ -62,7 +62,7 @@ def scan_binary(apkfile):
|
|||
usual_suspects = {
|
||||
# The `apkanalyzer dex packages` output looks like this:
|
||||
# M d 1 1 93 <packagename> <other stuff>
|
||||
# The first column has P/C/M/F for package, class, methos or field
|
||||
# The first column has P/C/M/F for package, class, method or field
|
||||
# The second column has x/k/r/d for removed, kept, referenced and defined.
|
||||
# We already filter for defined only in the apkanalyzer call. 'r' will be
|
||||
# for things referenced but not distributed in the apk.
|
||||
|
@ -177,6 +177,11 @@ def scan_source(build_dir, build=metadata.Build()):
|
|||
return False
|
||||
|
||||
def ignoreproblem(what, path_in_build_dir):
|
||||
"""
|
||||
:param what: string describing the problem, will be printed in log messages
|
||||
:param path_in_build_dir: path to the file relative to `build`-dir
|
||||
"returns: 0 as we explicitly ignore the file, so don't count an error
|
||||
"""
|
||||
msg = ('Ignoring %s at %s' % (what, path_in_build_dir))
|
||||
logging.info(msg)
|
||||
if json_per_build is not None:
|
||||
|
@ -184,14 +189,31 @@ def scan_source(build_dir, build=metadata.Build()):
|
|||
return 0
|
||||
|
||||
def removeproblem(what, path_in_build_dir, filepath):
|
||||
"""
|
||||
:param what: string describing the problem, will be printed in log messages
|
||||
:param path_in_build_dir: path to the file relative to `build`-dir
|
||||
:param filepath: Path (relative to our current path) to the file
|
||||
"returns: 0 as we deleted the offending file
|
||||
"""
|
||||
msg = ('Removing %s at %s' % (what, path_in_build_dir))
|
||||
logging.info(msg)
|
||||
if json_per_build is not None:
|
||||
json_per_build['infos'].append([msg, path_in_build_dir])
|
||||
os.remove(filepath)
|
||||
try:
|
||||
os.remove(filepath)
|
||||
except FileNotFoundError:
|
||||
# File is already gone, nothing to do.
|
||||
# This can happen if we find multiple problems in one file that is setup for scandelete
|
||||
# I.e. build.gradle files containig multiple unknown maven repos.
|
||||
pass
|
||||
return 0
|
||||
|
||||
def warnproblem(what, path_in_build_dir):
|
||||
"""
|
||||
:param what: string describing the problem, will be printed in log messages
|
||||
:param path_in_build_dir: path to the file relative to `build`-dir
|
||||
:returns: 0, as warnings don't count as errors
|
||||
"""
|
||||
if toignore(path_in_build_dir):
|
||||
return 0
|
||||
logging.warning('Found %s at %s' % (what, path_in_build_dir))
|
||||
|
@ -200,6 +222,14 @@ def scan_source(build_dir, build=metadata.Build()):
|
|||
return 0
|
||||
|
||||
def handleproblem(what, path_in_build_dir, filepath):
|
||||
"""Dispatches to problem handlers (ignore, delete, warn) or returns 1
|
||||
for increasing the error count
|
||||
|
||||
:param what: string describing the problem, will be printed in log messages
|
||||
:param path_in_build_dir: path to the file relative to `build`-dir
|
||||
:param filepath: Path (relative to our current path) to the file
|
||||
:returns: 0 if the problem was ignored/deleted/is only a warning, 1 otherwise
|
||||
"""
|
||||
if toignore(path_in_build_dir):
|
||||
return ignoreproblem(what, path_in_build_dir)
|
||||
if todelete(path_in_build_dir):
|
||||
|
|
|
@ -1974,12 +1974,6 @@ def apply_info_from_latest_apk(apps, apks):
|
|||
if app.NoSourceSince:
|
||||
apk['antiFeatures'].add('NoSourceSince')
|
||||
|
||||
if 'added' in apk:
|
||||
if not app.added or apk['added'] < app.added:
|
||||
app.added = apk['added']
|
||||
if not app.lastUpdated or apk['added'] > app.lastUpdated:
|
||||
app.lastUpdated = apk['added']
|
||||
|
||||
if not app.added:
|
||||
logging.debug("Don't know when " + appid + " was added")
|
||||
if not app.lastUpdated:
|
||||
|
@ -2166,6 +2160,27 @@ def create_metadata_from_template(apk):
|
|||
logging.info(_("Generated skeleton metadata for {appid}").format(appid=apk['packageName']))
|
||||
|
||||
|
||||
def read_added_date_from_all_apks(apps, apks):
|
||||
"""
|
||||
Added dates come from the stats/known_apks.txt file but are
|
||||
read when scanning apks and thus need to be applied form apk
|
||||
level to app level for _all_ apps and not only from non-archived
|
||||
ones
|
||||
|
||||
TODO: read the added dates directly from known_apks.txt instead of
|
||||
going through apks that way it also works for for repos that
|
||||
don't keep an archive of apks.
|
||||
"""
|
||||
for appid, app in apps.items():
|
||||
for apk in apks:
|
||||
if apk['packageName'] == appid:
|
||||
if 'added' in apk:
|
||||
if not app.added or apk['added'] < app.added:
|
||||
app.added = apk['added']
|
||||
if not app.lastUpdated or apk['added'] > app.lastUpdated:
|
||||
app.lastUpdated = apk['added']
|
||||
|
||||
|
||||
def read_names_from_apks(apps, apks):
|
||||
"""This is a stripped down copy of apply_info_from_latest_apk that only parses app names"""
|
||||
for appid, app in apps.items():
|
||||
|
@ -2384,6 +2399,10 @@ def main():
|
|||
# This will be done again (as part of apply_info_from_latest_apk) for repo and archive
|
||||
# separately later on, but it's fairly cheap anyway.
|
||||
read_names_from_apks(apps, apks + archapks)
|
||||
# The added date currently comes from the oldest apk which might be in the archive.
|
||||
# So we need this populated at app level before continuing with only processing /repo
|
||||
# or /archive
|
||||
read_added_date_from_all_apks(apps, apks + archapks)
|
||||
|
||||
if len(repodirs) > 1:
|
||||
archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
|
||||
|
|
|
@ -23,6 +23,7 @@ if localmodule not in sys.path:
|
|||
import fdroidserver.build
|
||||
import fdroidserver.common
|
||||
import fdroidserver.metadata
|
||||
import fdroidserver.scanner
|
||||
|
||||
|
||||
class BuildTest(unittest.TestCase):
|
||||
|
@ -198,6 +199,50 @@ class BuildTest(unittest.TestCase):
|
|||
self.assertFalse(os.path.exists('gen'))
|
||||
self.assertFalse(os.path.exists('gradle-wrapper.jar'))
|
||||
|
||||
def test_scan_with_extlib(self):
|
||||
testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
|
||||
os.chdir(testdir)
|
||||
os.mkdir("build")
|
||||
|
||||
config = dict()
|
||||
config['sdk_path'] = os.getenv('ANDROID_HOME')
|
||||
config['ndk_paths'] = {'r10d': os.getenv('ANDROID_NDK_HOME')}
|
||||
config['build_tools'] = 'FAKE_BUILD_TOOLS_VERSION'
|
||||
fdroidserver.common.config = config
|
||||
app = fdroidserver.metadata.App()
|
||||
app.id = 'com.gpl.rpg.AndorsTrail'
|
||||
build = fdroidserver.metadata.Build()
|
||||
build.commit = 'master'
|
||||
build.androidupdate = ['no']
|
||||
os.makedirs("extlib/android")
|
||||
# write a fake binary jar file the scanner should definitely error on
|
||||
with open('extlib/android/android-support-v4r11.jar', 'wb') as file:
|
||||
file.write(b'PK\x03\x04\x14\x00\x08\x00\x08\x00-\x0eiA\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\t\x00\x04\x00META-INF/\xfe\xca\x00\x00\x03\x00PK\x07\x08\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00')
|
||||
|
||||
class FakeVcs():
|
||||
# no need to change to the correct commit here
|
||||
def gotorevision(self, rev, refresh=True):
|
||||
pass
|
||||
|
||||
def getsrclib(self):
|
||||
return None
|
||||
|
||||
# Test we trigger a scanner error without extlibs
|
||||
build.extlibs = []
|
||||
os.makedirs('build/libs')
|
||||
shutil.copy('extlib/android/android-support-v4r11.jar', 'build/libs')
|
||||
fdroidserver.common.prepare_source(FakeVcs(), app, build,
|
||||
"build", "ignore", "extlib")
|
||||
count = fdroidserver.scanner.scan_source("build", build)
|
||||
self.assertEqual(1, count, "Should produce a scanner error without extlib")
|
||||
|
||||
# Now try again as an extlib
|
||||
build.extlibs = ['android/android-support-v4r11.jar']
|
||||
fdroidserver.common.prepare_source(FakeVcs(), app, build,
|
||||
"build", "ignore", "extlib")
|
||||
count = fdroidserver.scanner.scan_source("build", build)
|
||||
self.assertEqual(0, count, "Shouldn't error on jar from extlib")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
os.chdir(os.path.dirname(__file__))
|
||||
|
|
|
@ -607,6 +607,29 @@ grep -F 'info.guardianproject.urzip_100.apk' repo/index-v1.json repo/index.xml
|
|||
grep -F 'info.guardianproject.urzip_100_b4964fd.apk' repo/index-v1.json
|
||||
! grep -F 'info.guardianproject.urzip_100_b4964fd.apk' repo/index.xml
|
||||
|
||||
#------------------------------------------------------------------------------#
|
||||
echo_header "test for added date being set correctly for repo and archive"
|
||||
REPOROOT=`create_test_dir`
|
||||
cd $REPOROOT
|
||||
fdroid_init_with_prebuilt_keystore
|
||||
mkdir -p {repo,archive,metadata,stats}
|
||||
cp $WORKSPACE/tests/repo/com.politedroid_5.apk archive
|
||||
cp $WORKSPACE/tests/repo/com.politedroid_6.apk repo
|
||||
cp $WORKSPACE/tests/metadata/com.politedroid.yml metadata
|
||||
#TODO: the timestamp of the oldest apk in the file should be used, even if that
|
||||
# doesn't exist anymore
|
||||
echo "com.politedroid_4.apk com.politedroid 2016-01-01" > stats/known_apks.txt
|
||||
echo "com.politedroid_5.apk com.politedroid 2017-01-01" >> stats/known_apks.txt
|
||||
echo "com.politedroid_6.apk com.politedroid 2018-01-01" >> stats/known_apks.txt
|
||||
sed -i -e 's/ArchivePolicy:.*/ArchivePolicy: 1 versions/' metadata/com.politedroid.yml
|
||||
# Get the java ms timestamp from UTC time
|
||||
timestamp=$(date -u --date=2017-01-01 +%s)000
|
||||
|
||||
$fdroid update --pretty --nosign
|
||||
grep -F "\"added\": $timestamp" repo/index-v1.json
|
||||
# the archive will have the added timestamp for the app and for the apk, both need to be there
|
||||
if [ $(grep -F "\"added\": $timestamp" archive/index-v1.json | wc -l) == 2 ]; then true; else false;fi
|
||||
|
||||
#------------------------------------------------------------------------------#
|
||||
echo_header "test whatsnew from fastlane without CVC set"
|
||||
REPOROOT=`create_test_dir`
|
||||
|
|
|
@ -270,6 +270,27 @@ class ScannerTest(unittest.TestCase):
|
|||
self.assertTrue(found, 'this block should produce a URL:\n' + entry)
|
||||
self.assertEqual(len(data), len(urls), 'each data example should produce a URL')
|
||||
|
||||
def test_scan_gradle_file_with_multiple_problems(self):
|
||||
"""Check that the scanner can handle scandelete with gradle files with multiple problems"""
|
||||
testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
|
||||
os.chdir(testdir)
|
||||
fdroidserver.scanner.config = None
|
||||
fdroidserver.scanner.options = mock.Mock()
|
||||
build = fdroidserver.metadata.Build()
|
||||
build.scandelete = ['build.gradle']
|
||||
with open('build.gradle', 'w') as fp:
|
||||
fp.write(textwrap.dedent("""
|
||||
maven {
|
||||
url 'https://maven.fabric.io/public'
|
||||
}
|
||||
maven {
|
||||
url 'https://evilcorp.com/maven'
|
||||
}
|
||||
"""))
|
||||
count = fdroidserver.scanner.scan_source(testdir, build)
|
||||
self.assertFalse(os.path.exists("build.gradle"))
|
||||
self.assertEqual(0, count, 'there should be this many errors')
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
os.chdir(os.path.dirname(__file__))
|
||||
|
|
|
@ -0,0 +1,90 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
# http://www.drdobbs.com/testing/unit-testing-with-python/240165163
|
||||
|
||||
import inspect
|
||||
import logging
|
||||
import optparse
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from git import Repo
|
||||
|
||||
localmodule = os.path.realpath(
|
||||
os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..'))
|
||||
print('localmodule: ' + localmodule)
|
||||
if localmodule not in sys.path:
|
||||
sys.path.insert(0, localmodule)
|
||||
|
||||
import fdroidserver.build
|
||||
import fdroidserver.common
|
||||
import fdroidserver.metadata
|
||||
import fdroidserver.scanner
|
||||
|
||||
|
||||
class VCSTest(unittest.TestCase):
|
||||
"""For some reason the VCS classes are in fdroidserver/common.py"""
|
||||
|
||||
def setUp(self):
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
self.basedir = os.path.join(localmodule, 'tests')
|
||||
self.tmpdir = os.path.abspath(os.path.join(self.basedir, '..', '.testfiles'))
|
||||
if not os.path.exists(self.tmpdir):
|
||||
os.makedirs(self.tmpdir)
|
||||
os.chdir(self.basedir)
|
||||
|
||||
def test_remote_set_head_can_fail(self):
|
||||
testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
|
||||
os.chdir(testdir)
|
||||
# First create an upstream repo with one commit
|
||||
upstream_repo = Repo.init("upstream_repo")
|
||||
with open(upstream_repo.working_dir + "/file", 'w') as f:
|
||||
f.write("Hello World!")
|
||||
upstream_repo.index.add([upstream_repo.working_dir + "/file"])
|
||||
upstream_repo.index.commit("initial commit")
|
||||
commitid = upstream_repo.head.commit.hexsha
|
||||
|
||||
# Now clone it once manually, like gitlab runner gitlab-runner sets up a repo during CI
|
||||
clone1 = Repo.init("clone1")
|
||||
clone1.create_remote("upstream", "file://" + upstream_repo.working_dir)
|
||||
clone1.remote("upstream").fetch()
|
||||
clone1.head.reference = clone1.commit(commitid)
|
||||
clone1.head.reset(index=True, working_tree=True)
|
||||
self.assertTrue(clone1.head.is_detached)
|
||||
|
||||
# and now we want to use this clone as a source repo for fdroid build
|
||||
config = {}
|
||||
os.mkdir("build")
|
||||
config['sdk_path'] = os.getenv('ANDROID_HOME')
|
||||
config['ndk_paths'] = {'r10d': os.getenv('ANDROID_NDK_HOME')}
|
||||
config['build_tools'] = 'FAKE_BUILD_TOOLS_VERSION'
|
||||
config['java_paths'] = {'fake': 'fake'}
|
||||
fdroidserver.common.config = config
|
||||
app = fdroidserver.metadata.App()
|
||||
app.RepoType = 'git'
|
||||
app.Repo = clone1.working_dir
|
||||
app.id = 'com.gpl.rpg.AndorsTrail'
|
||||
build = fdroidserver.metadata.Build()
|
||||
build.commit = commitid
|
||||
build.androidupdate = ['no']
|
||||
vcs, build_dir = fdroidserver.common.setup_vcs(app)
|
||||
# force an init of the repo, the remote head error only occurs on the second gotorevision call
|
||||
vcs.gotorevision(build.commit)
|
||||
fdroidserver.common.prepare_source(vcs, app, build,
|
||||
build_dir=build_dir, srclib_dir="ignore", extlib_dir="ignore")
|
||||
self.assertTrue(os.path.isfile("build/com.gpl.rpg.AndorsTrail/file"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
os.chdir(os.path.dirname(__file__))
|
||||
|
||||
parser = optparse.OptionParser()
|
||||
parser.add_option("-v", "--verbose", action="store_true", default=False,
|
||||
help="Spew out even more information than normal")
|
||||
(fdroidserver.common.options, args) = parser.parse_args(['--verbose'])
|
||||
|
||||
newSuite = unittest.TestSuite()
|
||||
newSuite.addTest(unittest.makeSuite(VCSTest))
|
||||
unittest.main(failfast=False)
|
Loading…
Reference in New Issue