Merge branch 'androguard-modernization' into 'master'

port to androguard >= 4 and drop support for older than 3.3.5

See merge request fdroid/fdroidserver!1462

fdroidserver/common.py

@@ -2629,29 +2629,34 @@ def get_file_extension(filename):
     return os.path.splitext(filename)[1].lower()[1:]
 
 
-def use_androguard():
-    """Report if androguard is available, and config its debug logging."""
-    try:
-        import androguard
-        if use_androguard.show_path:
-            logging.debug(_('Using androguard from "{path}"').format(path=androguard.__file__))
-            use_androguard.show_path = False
-        if options and options.verbose:
-            logging.getLogger("androguard.axml").setLevel(logging.INFO)
-        logging.getLogger("androguard.core.api_specific_resources").setLevel(logging.ERROR)
-        return True
-    except ImportError:
-        return False
+def _androguard_logging_level(level=logging.ERROR):
+    """Tames androguard's default debug output.
 
+    There should be no debug output when the functions are being used
+    via the API.  Otherwise, the output is controlled by the --verbose
+    flag.
 
-use_androguard.show_path = True  # type: ignore
+    """
+    if options and options.verbose:
+        level = logging.WARNING
+
+    for name in (
+        'androguard.apk',
+        'androguard.axml',
+        'androguard.core.api_specific_resources',
+        'androguard.core.apk',
+        'androguard.core.axml',
+    ):
+        logging.getLogger(name).setLevel(level)
 
 
 def _get_androguard_APK(apkfile):
     try:
+        # these were moved in androguard 4.0
+        from androguard.core.apk import APK
+    except ImportError:
         from androguard.core.bytecodes.apk import APK
-    except ImportError as exc:
-        raise FDroidException("androguard library is not installed") from exc
+    _androguard_logging_level()
 
     return APK(apkfile)
 
@@ -2693,7 +2698,13 @@ def is_debuggable_or_testOnly(apkfile):
     """
     if get_file_extension(apkfile) != 'apk':
         return False
-    from androguard.core.bytecodes.axml import AXMLParser, format_value, START_TAG
+    try:
+        # these were moved in androguard 4.0
+        from androguard.core.axml import AXMLParser, format_value, START_TAG
+    except ImportError:
+        from androguard.core.bytecodes.axml import AXMLParser, format_value, START_TAG
+    _androguard_logging_level()
+
     with ZipFile(apkfile) as apk:
         with apk.open('AndroidManifest.xml') as manifest:
             axml = AXMLParser(manifest.read())
@@ -2753,12 +2764,20 @@ def get_apk_id_androguard(apkfile):
     versionName is set to a Android String Resource (e.g. an integer
     hex value that starts with @).
 
+    This function is part of androguard as get_apkid(), so this
+    vendored and modified to return versionCode as an integer.
+
     """
     if not os.path.exists(apkfile):
         raise FDroidException(_("Reading packageName/versionCode/versionName failed, APK invalid: '{apkfilename}'")
                               .format(apkfilename=apkfile))
 
-    from androguard.core.bytecodes.axml import AXMLParser, format_value, START_TAG, END_TAG, TEXT, END_DOCUMENT
+    try:
+        # these were moved in androguard 4.0
+        from androguard.core.axml import AXMLParser, format_value, START_TAG, END_TAG, TEXT, END_DOCUMENT
+    except ImportError:
+        from androguard.core.bytecodes.axml import AXMLParser, format_value, START_TAG, END_TAG, TEXT, END_DOCUMENT
+    _androguard_logging_level()
 
     appid = None
     versionCode = None
@@ -3159,7 +3178,7 @@ def get_first_signer_certificate(apkpath):
         elif len(cert_files) == 1:
             cert_encoded = get_certificate(apk.read(cert_files[0]))
 
-    if not cert_encoded and use_androguard():
+    if not cert_encoded:
         apkobject = _get_androguard_APK(apkpath)
         certs = apkobject.get_certificates_der_v2()
         if len(certs) > 0:

fdroidserver/update.py

@@ -1571,8 +1571,7 @@ def _sanitize_sdk_version(value):
 
 def scan_apk_androguard(apk, apkfile):
     try:
-        from androguard.core.bytecodes.apk import APK
-        apkobject = APK(apkfile)
+        apkobject = common._get_androguard_APK(apkfile)
         if apkobject.is_valid_APK():
             arsc = apkobject.get_android_resources()
         else:
@@ -1628,7 +1627,7 @@ def scan_apk_androguard(apk, apkfile):
     if maxSdkVersion is not None:
         apk['maxSdkVersion'] = maxSdkVersion
 
-    icon_id_str = apkobject.get_element("application", "icon")
+    icon_id_str = apkobject.get_attribute_value("application", "icon")
     if icon_id_str:
         try:
             icon_id = int(icon_id_str.replace("@", "0x"), 16)
@@ -2430,7 +2429,6 @@ def main():
     config = common.read_config(options)
     common.setup_status_output(start_timestamp)
 
-    common.use_androguard()
     if not (('jarsigner' in config or 'apksigner' in config)
             and 'keytool' in config):
         raise FDroidException(_('Java JDK not found! Install in standard location or set java_paths!'))

setup.py

@@ -92,7 +92,7 @@ setup(
     ],
     install_requires=[
         'appdirs',
-        'androguard >= 3.1.0, != 3.3.0, != 3.3.1, != 3.3.2, <4',
+        'androguard >= 3.3.5',
         'clint',
         'defusedxml',
         'GitPython',

tests/build.TestCase

@@ -49,8 +49,6 @@ class BuildTest(unittest.TestCase):
 
     def setUp(self):
         logging.basicConfig(level=logging.DEBUG)
-        logger = logging.getLogger('androguard.axml')
-        logger.setLevel(logging.INFO)  # tame the axml debug messages
         self.basedir = os.path.join(localmodule, 'tests')
         os.chdir(self.basedir)
         fdroidserver.common.config = None

tests/run-tests

@@ -8,7 +8,9 @@ echo_header() {
 
 get_fdroid_apk_filename() {
     if [ -z $aapt ]; then
-        python3 -c "from androguard.core.bytecodes.apk import APK; a=APK('$1'); print(a.package+'_'+a.get_androidversion_code()+'.apk')"
+        appid=$(androguard apkid "$1" | grep -Eio '"[a-z][^"]+",' | sed 's/[,"]//g')
+        versionCode=$(androguard apkid "$1" | grep -Eo '"[0-9]+"' | sed 's,",,g')
+        echo "${appid}_${versionCode}.apk"
     else
         $aapt dump badging "$1" | sed -n "s,^package: name='\(.*\)' versionCode='\([0-9][0-9]*\)' .*,\1_\2.apk,p"
     fi

tests/update.TestCase

@@ -5,6 +5,7 @@
 import copy
 import git
 import glob
+import hashlib
 import inspect
 import json
 import logging
@@ -20,11 +21,18 @@ import unittest
 import yaml
 import zipfile
 import textwrap
+from binascii import hexlify
 from datetime import datetime
 from pathlib import Path
 from testcommon import TmpCwd, mkdtemp
 from unittest import mock
 
+try:
+    # these were moved in androguard 4.0
+    from androguard.core.apk import APK
+except ImportError:
+    from androguard.core.bytecodes.apk import APK
+
 try:
     from yaml import CSafeLoader as SafeLoader
 except ImportError:
@@ -74,9 +82,6 @@ class UpdateTest(unittest.TestCase):
 
     def setUp(self):
         logging.basicConfig(level=logging.INFO)
-        logging.getLogger('androguard.apk').setLevel(logging.WARNING)
-        logging.getLogger('androguard.axml').setLevel(logging.INFO)
-        logging.getLogger('androguard.core.api_specific_resources').setLevel(logging.INFO)
         from PIL import PngImagePlugin
 
         logging.getLogger(PngImagePlugin.__name__).setLevel(logging.INFO)
@@ -581,13 +586,6 @@ class UpdateTest(unittest.TestCase):
         self.assertEqual(good_fingerprint, sig,
                          'python sig was: ' + str(sig))
         # check that v1 and v2 have the same certificate
-        try:
-            import hashlib
-            from binascii import hexlify
-            from androguard.core.bytecodes.apk import APK
-        except ImportError:
-            print('WARNING: skipping rest of test since androguard is missing!')
-            return
         apkobject = APK(apkpath)
         cert_encoded = apkobject.get_certificates_der_v2()[0]
         self.assertEqual(good_fingerprint, sig,