use UTF8 as default instead of ASCII for .java .gradle pom.xml

.java .gradle and XML files all can use any encoding.  Most code is ASCII,
but authors' names, etc. can easily be non-ASCII.  UTF-8 is by far the most
common file encoding.  While UTF-8 is the default encoding inside the code
in Python 3, it still has to deal with the real world, so the encoding
needs to be explicitly set when reading and writing files. So this switches
fdroidserver to expect UTF-8 instead of ASCII when parsing these files. For
now, this commit means that we only support UTF-8 encoded *.java, pom.xml
or *.gradle files.  Ideally, the code would detect the encoding and use the
actual one, but that's a lot more work, and its something that will not
happen often. We can cross that bridge when we come to it.

One approach, which is taken in the commit when possible, is to keep the
data as `bytes`, in which case the encoding doesn't matter.

This also fixes this crash when parsing gradle and maven files with
non-ASCII chars:

ERROR: test_adapt_gradle (__main__.BuildTest)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/var/lib/jenkins/workspace/fdroidserver-eighthave/tests/build.TestCase", line 59, in test_adapt_gradle
    fdroidserver.build.adapt_gradle(testsdir)
  File "/var/lib/jenkins/workspace/fdroidserver-eighthave/fdroidserver/build.py", line 445, in adapt_gradle
    path)
  File "/var/lib/jenkins/workspace/fdroidserver-eighthave/fdroidserver/common.py", line 188, in regsub_file
    text = f.read()
  File "/usr/lib/python3.4/encodings/ascii.py", line 26, in decode
    return codecs.ascii_decode(input, self.errors)[0]
UnicodeDecodeError: 'ascii' codec can't decode byte 0xe2 in position 9460: ordinal not in range(128)

fdroidserver/common.py

@@ -184,10 +184,10 @@ def fill_config_defaults(thisconfig):
 
 
 def regsub_file(pattern, repl, path):
-    with open(path, 'r') as f:
+    with open(path, 'rb') as f:
         text = f.read()
-    text = re.sub(pattern, repl, text)
-    with open(path, 'w') as f:
+    text = re.sub(bytes(pattern, 'utf8'), bytes(repl, 'utf8'), text)
+    with open(path, 'wb') as f:
         f.write(text)
 
 
@@ -1724,14 +1724,14 @@ def remove_signing_keys(build_dir):
         if 'build.gradle' in files:
             path = os.path.join(root, 'build.gradle')
 
-            with open(path, "r") as o:
+            with open(path, "r", encoding='utf8') as o:
                 lines = o.readlines()
 
             changed = False
 
             opened = 0
             i = 0
-            with open(path, "w") as o:
+            with open(path, "w", encoding='utf8') as o:
                 while i < len(lines):
                     line = lines[i]
                     i += 1

fdroidserver/metadata.py

@@ -836,7 +836,7 @@ def get_default_app_info(metadatapath=None):
             for root, dirs, files in os.walk(os.getcwd()):
                 if 'build.gradle' in files:
                     p = os.path.join(root, 'build.gradle')
-                    with open(p) as f:
+                    with open(p, 'rb') as f:
                         data = f.read()
                     m = pattern.search(data)
                     if m:

fdroidserver/scanner.py

@@ -199,7 +199,7 @@ def scan_source(build_dir, root_dir, build):
             elif ext == 'java':
                 if not os.path.isfile(fp):
                     continue
-                with open(fp, 'r') as f:
+                with open(fp, 'r', encoding='utf8') as f:
                     for line in f:
                         if 'DexClassLoader' in line:
                             count += handleproblem('DexClassLoader', fd, fp)
@@ -208,7 +208,7 @@ def scan_source(build_dir, root_dir, build):
             elif ext == 'gradle':
                 if not os.path.isfile(fp):
                     continue
-                with open(fp, 'r') as f:
+                with open(fp, 'r', encoding='utf8') as f:
                     lines = f.readlines()
                 for i, line in enumerate(lines):
                     if is_used_by_gradle(line):

tests/build.TestCase

@@ -57,13 +57,13 @@ class BuildTest(unittest.TestCase):
         fdroidserver.build.config = {}
         fdroidserver.build.config['build_tools'] = teststring
         fdroidserver.build.adapt_gradle(testsdir)
-        pattern = re.compile("buildToolsVersion[\s=]+'%s'\s+" % teststring)
+        pattern = re.compile(bytes("buildToolsVersion[\s=]+'%s'\s+" % teststring, 'utf8'))
         for p in ('source-files/fdroid/fdroidclient/build.gradle',
                   'source-files/Zillode/syncthing-silk/build.gradle',
                   'source-files/open-keychain/open-keychain/build.gradle',
                   'source-files/osmandapp/osmand/build.gradle',
                   'source-files/open-keychain/open-keychain/OpenKeychain/build.gradle'):
-            with open(os.path.join(testsdir, p), 'r') as f:
+            with open(os.path.join(testsdir, p), 'rb') as f:
                 filedata = f.read()
             self.assertIsNotNone(pattern.search(filedata))