opensourcemirror
/
fdroid-server
mirror of https://gitlab.com/fdroid/fdroidserver.git
1
0
Fork 0
Code Issues Releases Wiki Activity

net.download_file(): retry on errors

This commit is contained in:
FC Stegerman 2022-10-22 23:15:13 +02:00
parent c61c9f3895
commit d1ddd525c1
No known key found for this signature in database
GPG Key ID: B218FF2C27FC6CC6
2 changed files with 39 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
fdroidserver/net.py
View File

@ -2,6 +2,7 @@
#
# net.py - part of the FDroid server tools
# Copyright (C) 2015 Hans-Christoph Steiner <hans@eds.org>
# Copyright (C) 2022 FC Stegerman <flx@obfusk.net>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
@ -16,28 +17,51 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import logging
import os
import requests
import time
import urllib
from requests.adapters import HTTPAdapter, Retry
from requests.exceptions import ChunkedEncodingError
HEADERS = {'User-Agent': 'F-Droid'}
def download_file(url, local_filename=None, dldir='tmp'):
def download_file(url, local_filename=None, dldir='tmp', retries=3, backoff_factor=0.1):
filename = urllib.parse.urlparse(url).path.split('/')[-1]
if local_filename is None:
local_filename = os.path.join(dldir, filename)
# the stream=True parameter keeps memory usage low
r = requests.get(
url, stream=True, allow_redirects=True, headers=HEADERS, timeout=300
)
r.raise_for_status()
with open(local_filename, 'wb') as f:
for chunk in r.iter_content(chunk_size=1024):
if chunk: # filter out keep-alive new chunks
f.write(chunk)
f.flush()
return local_filename
# Retry applies to failed DNS lookups, socket connections and connection
# timeouts, never to requests where data has made it to the server; so we
# handle ChunkedEncodingError during transfer ourselves.
for i in range(retries + 1):
if retries:
max_retries = Retry(total=retries - i, backoff_factor=backoff_factor)
adapter = HTTPAdapter(max_retries=max_retries)
session = requests.Session()
session.mount('http://', adapter)
session.mount('https://', adapter)
else:
session = requests
# the stream=True parameter keeps memory usage low
r = session.get(
url, stream=True, allow_redirects=True, headers=HEADERS, timeout=300
)
r.raise_for_status()
try:
with open(local_filename, 'wb') as f:
for chunk in r.iter_content(chunk_size=1024):
if chunk: # filter out keep-alive new chunks
f.write(chunk)
f.flush()
return local_filename
except ChunkedEncodingError as err:
if i == retries:
raise err
logging.warning('Download interrupted, retrying...')
time.sleep(backoff_factor * 2**i)
raise ValueError("retries must be >= 0")
def http_get(url, etag=None, timeout=600):

5
tests/net.TestCase
View File

@ -39,13 +39,14 @@ class NetTest(unittest.TestCase):
return MagicMock()
requests_get.side_effect = _get
f = net.download_file('https://f-droid.org/repo/index-v1.jar')
f = net.download_file('https://f-droid.org/repo/index-v1.jar', retries=0)
self.assertTrue(requests_get.called)
self.assertTrue(os.path.exists(f))
self.assertEqual('tmp/index-v1.jar', f)
f = net.download_file(
'https://d-05.example.com/custom/com.downloader.aegis-3175421.apk?_fn=QVBLUHVyZV92My4xNy41NF9hcGtwdXJlLmNvbS5hcGs&_p=Y29tLmFwa3B1cmUuYWVnb24&am=6avvTpfJ1dMl9-K6JYKzQw&arg=downloader%3A%2F%2Fcampaign%2F%3Futm_medium%3Ddownloader%26utm_source%3Daegis&at=1652080635&k=1f6e58465df3a441665e585719ab0b13627a117f&r=https%3A%2F%2Fdownloader.com%2Fdownloader-app.html%3Ficn%3Daegis%26ici%3Dimage_qr&uu=http%3A%2F%2F172.16.82.1%2Fcustom%2Fcom.downloader.aegis-3175421.apk%3Fk%3D3fb9c4ae0be578206f6a1c330736fac1627a117f'
'https://d-05.example.com/custom/com.downloader.aegis-3175421.apk?_fn=QVBLUHVyZV92My4xNy41NF9hcGtwdXJlLmNvbS5hcGs&_p=Y29tLmFwa3B1cmUuYWVnb24&am=6avvTpfJ1dMl9-K6JYKzQw&arg=downloader%3A%2F%2Fcampaign%2F%3Futm_medium%3Ddownloader%26utm_source%3Daegis&at=1652080635&k=1f6e58465df3a441665e585719ab0b13627a117f&r=https%3A%2F%2Fdownloader.com%2Fdownloader-app.html%3Ficn%3Daegis%26ici%3Dimage_qr&uu=http%3A%2F%2F172.16.82.1%2Fcustom%2Fcom.downloader.aegis-3175421.apk%3Fk%3D3fb9c4ae0be578206f6a1c330736fac1627a117f',
retries=0
)
self.assertTrue(requests_get.called)
self.assertTrue(os.path.exists(f))