Support ETag when downloading repository index

This commit is contained in:
Torsten Grote 2017-05-02 12:05:48 -03:00
parent e7e97654b1
commit 8d424f19ec
No known key found for this signature in database
GPG Key ID: 3E5F77D92CF891FF
4 changed files with 84 additions and 13 deletions

View File

@ -35,9 +35,7 @@ from binascii import hexlify, unhexlify
from datetime import datetime from datetime import datetime
from xml.dom.minidom import Document from xml.dom.minidom import Document
import requests from fdroidserver import metadata, signindex, common, net
from fdroidserver import metadata, signindex, common
from fdroidserver.common import FDroidPopen, FDroidPopenBytes from fdroidserver.common import FDroidPopen, FDroidPopenBytes
from fdroidserver.metadata import MetaDataException from fdroidserver.metadata import MetaDataException
@ -557,14 +555,16 @@ class VerificationException(Exception):
pass pass
def download_repo_index(url_str, verify_fingerprint=True): def download_repo_index(url_str, etag=None, verify_fingerprint=True):
""" """
Downloads the repository index from the given :param url_str Downloads the repository index from the given :param url_str
and verifies the repository's fingerprint if :param verify_fingerprint is not False. and verifies the repository's fingerprint if :param verify_fingerprint is not False.
:raises: VerificationException() if the repository could not be verified :raises: VerificationException() if the repository could not be verified
:return: The index in JSON format. :return: A tuple consisting of:
- The index in JSON format or None if the index did not change
- The new eTag as returned by the HTTP request
""" """
url = urllib.parse.urlsplit(url_str) url = urllib.parse.urlsplit(url_str)
@ -576,11 +576,14 @@ def download_repo_index(url_str, verify_fingerprint=True):
fingerprint = query['fingerprint'][0] fingerprint = query['fingerprint'][0]
url = urllib.parse.SplitResult(url.scheme, url.netloc, url.path + '/index-v1.jar', '', '') url = urllib.parse.SplitResult(url.scheme, url.netloc, url.path + '/index-v1.jar', '', '')
r = requests.get(url.geturl()) download, new_etag = net.http_get(url.geturl(), etag)
if download is None:
return None, new_etag
with tempfile.NamedTemporaryFile() as fp: with tempfile.NamedTemporaryFile() as fp:
# write and open JAR file # write and open JAR file
fp.write(r.content) fp.write(download)
jar = zipfile.ZipFile(fp) jar = zipfile.ZipFile(fp)
# verify that the JAR signature is valid # verify that the JAR signature is valid
@ -601,7 +604,7 @@ def download_repo_index(url_str, verify_fingerprint=True):
# turn the apps into App objects # turn the apps into App objects
index["apps"] = [metadata.App(app) for app in index["apps"]] index["apps"] = [metadata.App(app) for app in index["apps"]]
return index return index, new_etag
def verify_jar_signature(file): def verify_jar_signature(file):

View File

@ -34,3 +34,34 @@ def download_file(url, local_filename=None, dldir='tmp'):
f.write(chunk) f.write(chunk)
f.flush() f.flush()
return local_filename return local_filename
def http_get(url, etag=None):
"""
Downloads the content from the given URL by making a GET request.
If an ETag is given, it will do a HEAD request first, to see if the content changed.
:param url: The URL to download from.
:param etag: The last ETag to be used for the request (optional).
:return: A tuple consisting of:
- The raw content that was downloaded or None if it did not change
- The new eTag as returned by the HTTP request
"""
headers = {'User-Agent': 'F-Droid'}
# TODO disable TLS Session IDs and TLS Session Tickets
# (plain text cookie visible to anyone who can see the network traffic)
if etag:
r = requests.head(url, headers=headers)
r.raise_for_status()
if 'ETag' in r.headers and etag == r.headers['ETag']:
return None, etag
r = requests.get(url, headers=headers)
r.raise_for_status()
new_etag = None
if 'ETag' in r.headers:
new_etag = r.headers['ETag']
return r.content, new_etag

View File

@ -6,6 +6,9 @@ import os
import sys import sys
import unittest import unittest
import zipfile import zipfile
from unittest.mock import patch
import requests
localmodule = os.path.realpath( localmodule = os.path.realpath(
os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..')) os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..'))
@ -18,6 +21,9 @@ import fdroidserver.index
import fdroidserver.signindex import fdroidserver.signindex
GP_FINGERPRINT = 'B7C2EEFD8DAC7806AF67DFCD92EB18126BC08312A7F2D6F3862E46013C7A6135'
class IndexTest(unittest.TestCase): class IndexTest(unittest.TestCase):
def setUp(self): def setUp(self):
@ -55,9 +61,7 @@ class IndexTest(unittest.TestCase):
'818E469465F96B704E27BE2FEE4C63AB' + '818E469465F96B704E27BE2FEE4C63AB' +
'9F83DDF30E7A34C7371A4728D83B0BC1') '9F83DDF30E7A34C7371A4728D83B0BC1')
if f == 'guardianproject.jar': if f == 'guardianproject.jar':
self.assertTrue(fingerprint == self.assertTrue(fingerprint == GP_FINGERPRINT)
'B7C2EEFD8DAC7806AF67DFCD92EB1812' +
'6BC08312A7F2D6F3862E46013C7A6135')
def test_get_public_key_from_jar_fails(self): def test_get_public_key_from_jar_fails(self):
basedir = os.path.dirname(__file__) basedir = os.path.dirname(__file__)
@ -72,10 +76,43 @@ class IndexTest(unittest.TestCase):
fdroidserver.index.download_repo_index("http://example.org") fdroidserver.index.download_repo_index("http://example.org")
def test_download_repo_index_no_jar(self): def test_download_repo_index_no_jar(self):
with self.assertRaises(zipfile.BadZipFile): with self.assertRaises(requests.exceptions.HTTPError):
fdroidserver.index.download_repo_index("http://example.org?fingerprint=nope") fdroidserver.index.download_repo_index("http://example.org?fingerprint=nope")
# TODO test_download_repo_index with an actual repository @patch('requests.head')
def test_download_repo_index_same_etag(self, head):
url = 'http://example.org?fingerprint=test'
etag = '"4de5-54d840ce95cb9"'
head.return_value.headers = {'ETag': etag}
index, new_etag = fdroidserver.index.download_repo_index(url, etag=etag)
self.assertIsNone(index)
self.assertEqual(etag, new_etag)
@patch('requests.get')
@patch('requests.head')
def test_download_repo_index_new_etag(self, head, get):
url = 'http://example.org?fingerprint=' + GP_FINGERPRINT
etag = '"4de5-54d840ce95cb9"'
# fake HTTP answers
head.return_value.headers = {'ETag': 'new_etag'}
get.return_value.headers = {'ETag': 'new_etag'}
get.return_value.status_code = 200
testfile = os.path.join(os.path.dirname(__file__), 'signindex', 'guardianproject-v1.jar')
with open(testfile, 'rb') as file:
get.return_value.content = file.read()
index, new_etag = fdroidserver.index.download_repo_index(url, etag=etag)
# assert that the index was retrieved properly
self.assertEqual('Guardian Project Official Releases', index['repo']['name'])
self.assertEqual(GP_FINGERPRINT, index['repo']['fingerprint'])
self.assertTrue(len(index['repo']['pubkey']) > 500)
self.assertEqual(10, len(index['apps']))
self.assertEqual(10, len(index['packages']))
self.assertEqual('new_etag', new_etag)
if __name__ == "__main__": if __name__ == "__main__":

Binary file not shown.