metadata: always open metadata files as UTF-8

Windows seems to require this, otherwise this happens:

Traceback (most recent call last):

  File "tests/update.TestCase", line 737, in test_translate_per_build_anti_features

    apps = fdroidserver.metadata.read_metadata(xref=True)

  File "C:\Users\travis\build\fdroidtravis\fdroidserver\fdroidserver\metadata.py", line 813, in read_metadata

    app = parse_metadata(metadatapath, appid in check_vcs, refresh)

  File "C:\Users\travis\build\fdroidtravis\fdroidserver\fdroidserver\metadata.py", line 1023, in parse_metadata

    parse_yaml_metadata(mf, app)

  File "C:\Users\travis\build\fdroidtravis\fdroidserver\fdroidserver\metadata.py", line 1073, in parse_yaml_metadata

    yamldata = yaml.safe_load(mf)

  File "C:\python37\lib\site-packages\yaml\__init__.py", line 162, in safe_load

    return load(stream, SafeLoader)

  File "C:\python37\lib\site-packages\yaml\__init__.py", line 112, in load

    loader = Loader(stream)

  File "C:\python37\lib\site-packages\yaml\loader.py", line 34, in __init__

    Reader.__init__(self, stream)

  File "C:\python37\lib\site-packages\yaml\reader.py", line 85, in __init__

    self.determine_encoding()

  File "C:\python37\lib\site-packages\yaml\reader.py", line 124, in determine_encoding

    self.update_raw()

  File "C:\python37\lib\site-packages\yaml\reader.py", line 178, in update_raw

    data = self.stream.read(size)

  File "C:\python37\lib\encodings\cp1252.py", line 23, in decode

    return codecs.charmap_decode(input,self.errors,decoding_table)[0]

UnicodeDecodeError: 'charmap' codec can't decode byte 0x81 in position 37: character maps to <undefined>

fdroidserver/common.py

@@ -1859,7 +1859,7 @@ def get_gradle_subdir(build_dir, paths):
         if not first_gradle_dir:
             first_gradle_dir = path.parent.relative_to(build_dir)
         if path.exists() and SETTINGS_GRADLE_REGEX.match(str(path.name)):
-            for m in GRADLE_SUBPROJECT_REGEX.finditer(path.read_text()):
+            for m in GRADLE_SUBPROJECT_REGEX.finditer(path.read_text(encoding='utf-8')):
                 for f in (path.parent / m.group(1)).glob('build.gradle*'):
                     with f.open() as fp:
                         for line in fp.readlines():

fdroidserver/metadata.py

@@ -482,7 +482,7 @@ def parse_yaml_srclib(metadatapath):
             if type(data) is not dict:
                 if platform.system() == 'Windows':
                     # Handle symlink on Windows
-                    symlink = metadatapath.parent / metadatapath.read_text()
+                    symlink = metadatapath.parent / metadatapath.read_text(encoding='utf-8')
                     if symlink.is_file():
                         with symlink.open("r", encoding="utf-8") as s:
                             data = yaml.load(s, Loader=SafeLoader)
@@ -740,7 +740,7 @@ def parse_metadata(metadatapath):
         app.id = name
 
     if metadatapath.suffix == '.yml':
-        with metadatapath.open('r') as mf:
+        with metadatapath.open('r', encoding='utf-8') as mf:
             parse_yaml_metadata(mf, app)
     else:
         _warn_or_exception(_('Unknown metadata format: {path} (use: *.yml)')

fdroidserver/rewritemeta.py

@@ -36,7 +36,7 @@ def proper_format(app):
     s = io.StringIO()
     # TODO: currently reading entire file again, should reuse first
     # read in metadata.py
-    cur_content = Path(app.metadatapath).read_text()
+    cur_content = Path(app.metadatapath).read_text(encoding='utf-8')
     if Path(app.metadatapath).suffix == '.yml':
         metadata.write_yaml(s, app)
     content = s.getvalue()

tests/metadata.TestCase

@@ -231,8 +231,8 @@ class MetadataTest(unittest.TestCase):
             self.maxDiff = None
             file_name = 'fake.ota.update.yml'
             self.assertEqual(
-                (testdir / file_name).read_text(),
-                (Path('metadata-rewrite-yml') / file_name).read_text(),
+                (testdir / file_name).read_text(encoding='utf-8'),
+                (Path('metadata-rewrite-yml') / file_name).read_text(encoding='utf-8'),
             )
 
     def test_rewrite_yaml_fdroidclient(self):
@@ -253,8 +253,8 @@ class MetadataTest(unittest.TestCase):
             self.maxDiff = None
             file_name = 'org.fdroid.fdroid.yml'
             self.assertEqual(
-                (testdir / file_name).read_text(),
-                (Path('metadata-rewrite-yml') / file_name).read_text(),
+                (testdir / file_name).read_text(encoding='utf-8'),
+                (Path('metadata-rewrite-yml') / file_name).read_text(encoding='utf-8'),
             )
 
     def test_rewrite_yaml_special_build_params(self):
@@ -274,8 +274,8 @@ class MetadataTest(unittest.TestCase):
             self.maxDiff = None
             file_name = 'app.with.special.build.params.yml'
             self.assertEqual(
-                (testdir / file_name).read_text(),
-                (Path('metadata-rewrite-yml') / file_name).read_text(),
+                (testdir / file_name).read_text(encoding='utf-8'),
+                (Path('metadata-rewrite-yml') / file_name).read_text(encoding='utf-8'),
             )
 
     def test_post_parse_yaml_metadata(self):

tests/rewritemeta.TestCase

@@ -48,7 +48,7 @@ class RewriteMetaTest(unittest.TestCase):
             rewritemeta.main()
 
             self.assertEqual(
-                Path('metadata/a.yml').read_text(),
+                Path('metadata/a.yml').read_text(encoding='utf-8'),
                 textwrap.dedent(
                     '''\
                     License: Unknown
@@ -62,7 +62,7 @@ class RewriteMetaTest(unittest.TestCase):
             )
 
             self.assertEqual(
-                Path('metadata/b.yml').read_text(),
+                Path('metadata/b.yml').read_text(encoding='utf-8'),
                 textwrap.dedent(
                     '''\
                     License: Unknown
@@ -90,7 +90,7 @@ class RewriteMetaTest(unittest.TestCase):
                     rewritemeta.main()
 
             self.assertEqual(
-                Path('metadata/a.yml').read_text(), 'AutoName: a'
+                Path('metadata/a.yml').read_text(encoding='utf-8'), 'AutoName: a'
             )