fdroid-client/download/src/androidMain/kotlin/org/fdroid/download/HttpDownloader.kt

183 lines
7.4 KiB
Kotlin

/*
* Copyright (C) 2014-2017 Peter Serwylo <peter@serwylo.com>
* Copyright (C) 2014-2018 Hans-Christoph Steiner <hans@eds.org>
* Copyright (C) 2015-2016 Daniel Martí <mvdan@mvdan.cc>
* Copyright (c) 2018 Senecto Limited
* Copyright (C) 2022 Torsten Grote <t at grobox.de>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 3
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.fdroid.download
import android.annotation.TargetApi
import android.os.Build.VERSION.SDK_INT
import io.ktor.client.features.ResponseException
import kotlinx.coroutines.DelicateCoroutinesApi
import kotlinx.coroutines.runBlocking
import mu.KotlinLogging
import java.io.File
import java.io.IOException
import java.io.InputStream
import java.util.Date
/**
* Download files over HTTP, with support for proxies, `.onion` addresses, HTTP Basic Auth, etc.
*/
public class HttpDownloader constructor(
private val httpManager: HttpManager,
private val request: DownloadRequest,
destFile: File,
) : Downloader(destFile) {
private companion object {
val log = KotlinLogging.logger {}
}
private var hasChanged = false
private var fileSize = -1L
override fun getInputStream(resumable: Boolean): InputStream {
throw NotImplementedError("Use getInputStreamSuspend instead.")
}
@Throws(IOException::class, NoResumeException::class)
override suspend fun getBytes(resumable: Boolean, receiver: (ByteArray) -> Unit) {
val skipBytes = if (resumable) outputFile.length() else null
return try {
httpManager.get(request, skipBytes, receiver)
} catch (e: ResponseException) {
throw IOException(e)
}
}
/**
* Get a remote file, checking the HTTP response code, if it has changed since
* the last time a download was tried.
*
*
* If the `ETag` does not match, it could be caused by the previous
* download of the same file coming from a mirror running on a different
* webserver, e.g. Apache vs Nginx. `Content-Length` and
* `Last-Modified` are used to check whether the file has changed since
* those are more standardized than `ETag`. Plus, Nginx and Apache 2.4
* defaults use only those two values to generate the `ETag` anyway.
* Unfortunately, other webservers and CDNs have totally different methods
* for generating the `ETag`. And mirrors that are syncing using a
* method other than `rsync` could easily have different `Last-Modified`
* times on the exact same file. On top of that, some services like GitHub's
* raw file support `raw.githubusercontent.com` and GitLab's raw file
* support do not set the `Last-Modified` header at all. So ultimately,
* then `ETag` needs to be used first and foremost, then this calculated
* `ETag` can serve as a common fallback.
*
*
* In order to prevent the `ETag` from being used as a form of tracking
* cookie, this code never sends the `ETag` to the server. Instead, it
* uses a `HEAD` request to get the `ETag` from the server, then
* only issues a `GET` if the `ETag` has changed.
*
*
* This uses a integer value for `Last-Modified` to avoid enabling the
* use of that value as some kind of "cookieless cookie". One second time
* resolution should be plenty since these files change more on the time
* space of minutes or hours.
*
* @see [update index from any available mirror](https://gitlab.com/fdroid/fdroidclient/issues/1708)
*
* @see [Cookieless cookies](http://lucb1e.com/rp/cookielesscookies)
*/
@OptIn(DelicateCoroutinesApi::class)
@Throws(IOException::class, InterruptedException::class)
override fun download() {
val headInfo = runBlocking {
httpManager.head(request, cacheTag) ?: throw IOException()
}
val expectedETag = cacheTag
cacheTag = headInfo.eTag
fileSize = headInfo.contentLength ?: -1
// If the ETag does not match, it could be because the file is on a mirror
// running a different webserver, e.g. Apache vs Nginx.
// Content-Length and Last-Modified could be used as well.
// Nginx and Apache 2.4 defaults use only those two values to generate the ETag.
// Unfortunately, other webservers and CDNs have totally different methods.
// And mirrors that are syncing using a method other than rsync
// could easily have different Last-Modified times on the exact same file.
// On top of that, some services like GitHub's and GitLab's raw file support
// do not set the header at all.
val lastModified = try {
// this method is not available multi-platform, so for now only done in JVM
@Suppress("Deprecation")
Date.parse(headInfo.lastModified) / 1000
} catch (e: Exception) {
0L
}
val calculatedEtag: String =
String.format("%x-%x", lastModified, headInfo.contentLength)
// !headInfo.eTagChanged: expectedETag == headInfo.eTag (the expected ETag was in server response)
// calculatedEtag == expectedETag (ETag calculated from server response matches expected ETag)
if (!headInfo.eTagChanged || calculatedEtag == expectedETag) {
// ETag has not changed, don't download again
log.debug { "${request.path} cached, not downloading." }
hasChanged = false
return
}
hasChanged = true
var resumable = false
val fileLength = outputFile.length()
if (fileLength > fileSize) {
if (!outputFile.delete()) log.warn {
"Warning: " + outputFile.absolutePath + " not deleted"
}
} else if (fileLength == fileSize && outputFile.isFile) {
log.debug { "Already have outputFile, not download. ${outputFile.absolutePath}" }
return // already have it!
} else if (fileLength > 0) {
resumable = true
}
log.debug { "downloading ${request.path} (is resumable: $resumable)" }
runBlocking {
try {
downloadFromBytesReceiver(resumable)
} catch (e: NoResumeException) {
require(resumable) { "Got $e even though download was not resumable" }
if (!outputFile.delete()) log.warn {
"Warning: " + outputFile.absolutePath + " not deleted"
}
downloadFromBytesReceiver(false)
}
}
}
@TargetApi(24)
public override fun totalDownloadSize(): Long {
return if (SDK_INT < 24) {
fileSize.toInt().toLong() // TODO why?
} else {
fileSize
}
}
override fun hasChanged(): Boolean {
return hasChanged
}
override fun close() {
}
}