From a6d35a7ee115cc92c4ed971ce1db1deaa9503532 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 25 Sep 2019 13:34:11 +0200 Subject: [PATCH] metadata: always open metadata files as UTF-8 Windows seems to require this, otherwise this happens: Traceback (most recent call last): File "tests/update.TestCase", line 737, in test_translate_per_build_anti_features apps = fdroidserver.metadata.read_metadata(xref=True) File "C:\Users\travis\build\fdroidtravis\fdroidserver\fdroidserver\metadata.py", line 813, in read_metadata app = parse_metadata(metadatapath, appid in check_vcs, refresh) File "C:\Users\travis\build\fdroidtravis\fdroidserver\fdroidserver\metadata.py", line 1023, in parse_metadata parse_yaml_metadata(mf, app) File "C:\Users\travis\build\fdroidtravis\fdroidserver\fdroidserver\metadata.py", line 1073, in parse_yaml_metadata yamldata = yaml.safe_load(mf) File "C:\python37\lib\site-packages\yaml\__init__.py", line 162, in safe_load return load(stream, SafeLoader) File "C:\python37\lib\site-packages\yaml\__init__.py", line 112, in load loader = Loader(stream) File "C:\python37\lib\site-packages\yaml\loader.py", line 34, in __init__ Reader.__init__(self, stream) File "C:\python37\lib\site-packages\yaml\reader.py", line 85, in __init__ self.determine_encoding() File "C:\python37\lib\site-packages\yaml\reader.py", line 124, in determine_encoding self.update_raw() File "C:\python37\lib\site-packages\yaml\reader.py", line 178, in update_raw data = self.stream.read(size) File "C:\python37\lib\encodings\cp1252.py", line 23, in decode return codecs.charmap_decode(input,self.errors,decoding_table)[0] UnicodeDecodeError: 'charmap' codec can't decode byte 0x81 in position 37: character maps to --- fdroidserver/common.py | 2 +- fdroidserver/metadata.py | 4 ++-- fdroidserver/rewritemeta.py | 2 +- tests/metadata.TestCase | 12 ++++++------ tests/rewritemeta.TestCase | 6 +++--- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 09b66c44..2392be9a 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -1859,7 +1859,7 @@ def get_gradle_subdir(build_dir, paths): if not first_gradle_dir: first_gradle_dir = path.parent.relative_to(build_dir) if path.exists() and SETTINGS_GRADLE_REGEX.match(str(path.name)): - for m in GRADLE_SUBPROJECT_REGEX.finditer(path.read_text()): + for m in GRADLE_SUBPROJECT_REGEX.finditer(path.read_text(encoding='utf-8')): for f in (path.parent / m.group(1)).glob('build.gradle*'): with f.open() as fp: for line in fp.readlines(): diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index b3dc1fb2..59b15472 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -482,7 +482,7 @@ def parse_yaml_srclib(metadatapath): if type(data) is not dict: if platform.system() == 'Windows': # Handle symlink on Windows - symlink = metadatapath.parent / metadatapath.read_text() + symlink = metadatapath.parent / metadatapath.read_text(encoding='utf-8') if symlink.is_file(): with symlink.open("r", encoding="utf-8") as s: data = yaml.load(s, Loader=SafeLoader) @@ -740,7 +740,7 @@ def parse_metadata(metadatapath): app.id = name if metadatapath.suffix == '.yml': - with metadatapath.open('r') as mf: + with metadatapath.open('r', encoding='utf-8') as mf: parse_yaml_metadata(mf, app) else: _warn_or_exception(_('Unknown metadata format: {path} (use: *.yml)') diff --git a/fdroidserver/rewritemeta.py b/fdroidserver/rewritemeta.py index 172359b8..a6406cb9 100644 --- a/fdroidserver/rewritemeta.py +++ b/fdroidserver/rewritemeta.py @@ -36,7 +36,7 @@ def proper_format(app): s = io.StringIO() # TODO: currently reading entire file again, should reuse first # read in metadata.py - cur_content = Path(app.metadatapath).read_text() + cur_content = Path(app.metadatapath).read_text(encoding='utf-8') if Path(app.metadatapath).suffix == '.yml': metadata.write_yaml(s, app) content = s.getvalue() diff --git a/tests/metadata.TestCase b/tests/metadata.TestCase index 76094724..07d8edcf 100755 --- a/tests/metadata.TestCase +++ b/tests/metadata.TestCase @@ -231,8 +231,8 @@ class MetadataTest(unittest.TestCase): self.maxDiff = None file_name = 'fake.ota.update.yml' self.assertEqual( - (testdir / file_name).read_text(), - (Path('metadata-rewrite-yml') / file_name).read_text(), + (testdir / file_name).read_text(encoding='utf-8'), + (Path('metadata-rewrite-yml') / file_name).read_text(encoding='utf-8'), ) def test_rewrite_yaml_fdroidclient(self): @@ -253,8 +253,8 @@ class MetadataTest(unittest.TestCase): self.maxDiff = None file_name = 'org.fdroid.fdroid.yml' self.assertEqual( - (testdir / file_name).read_text(), - (Path('metadata-rewrite-yml') / file_name).read_text(), + (testdir / file_name).read_text(encoding='utf-8'), + (Path('metadata-rewrite-yml') / file_name).read_text(encoding='utf-8'), ) def test_rewrite_yaml_special_build_params(self): @@ -274,8 +274,8 @@ class MetadataTest(unittest.TestCase): self.maxDiff = None file_name = 'app.with.special.build.params.yml' self.assertEqual( - (testdir / file_name).read_text(), - (Path('metadata-rewrite-yml') / file_name).read_text(), + (testdir / file_name).read_text(encoding='utf-8'), + (Path('metadata-rewrite-yml') / file_name).read_text(encoding='utf-8'), ) def test_post_parse_yaml_metadata(self): diff --git a/tests/rewritemeta.TestCase b/tests/rewritemeta.TestCase index 1271f9d8..309e2f45 100755 --- a/tests/rewritemeta.TestCase +++ b/tests/rewritemeta.TestCase @@ -48,7 +48,7 @@ class RewriteMetaTest(unittest.TestCase): rewritemeta.main() self.assertEqual( - Path('metadata/a.yml').read_text(), + Path('metadata/a.yml').read_text(encoding='utf-8'), textwrap.dedent( '''\ License: Unknown @@ -62,7 +62,7 @@ class RewriteMetaTest(unittest.TestCase): ) self.assertEqual( - Path('metadata/b.yml').read_text(), + Path('metadata/b.yml').read_text(encoding='utf-8'), textwrap.dedent( '''\ License: Unknown @@ -90,7 +90,7 @@ class RewriteMetaTest(unittest.TestCase): rewritemeta.main() self.assertEqual( - Path('metadata/a.yml').read_text(), 'AutoName: a' + Path('metadata/a.yml').read_text(encoding='utf-8'), 'AutoName: a' )