opensourcemirror
/
fdroid-client
mirror of https://gitlab.com/fdroid/fdroidclient.git
1
0
Fork 0
Code Issues Releases Wiki Activity

[db] make repo certs non-null and remove repos without cert 

historically, repos were added to the DB without much information and could stay in a broken state until manually removed. If a repo is updating, it should have a cert. So only repos that never did a single index update don't have a cert. Nowadays, this can not happen anymore as we get the repo and its cert before adding it to the DB. So whenever we update a repo, we know its certificate and fingerprint. Thus, this includes a DB migration removing all broken repos and making the certificate for repos in the DB non-null.
This commit is contained in:
Torsten Grote 2024-04-05 18:04:52 -03:00
parent fbf96a80cd
commit afd11f285f
No known key found for this signature in database
GPG Key ID: 3E5F77D92CF891FF
26 changed files with 2373 additions and 168 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1098
libs/database/schemas/org.fdroid.database.FDroidDatabaseInt/3.json Normal file
View File

File diff suppressed because it is too large Load Diff

1098
libs/database/schemas/org.fdroid.database.FDroidDatabaseInt/4.json Normal file
View File

File diff suppressed because it is too large Load Diff

8
libs/database/src/dbTest/java/org/fdroid/database/DbTest.kt
View File

@ -76,9 +76,9 @@ internal abstract class DbTest {
certificate: String = CERTIFICATE,
lastTimestamp: Long = -1,
): Long {
val repoId = db.getRepositoryDao().insertEmptyRepo(address)
val repoId = db.getRepositoryDao().insertEmptyRepo(address, certificate = certificate)
val streamReceiver = DbV1StreamReceiver(db, repoId) { true }
val indexProcessor = IndexV1StreamProcessor(streamReceiver, certificate, lastTimestamp)
val indexProcessor = IndexV1StreamProcessor(streamReceiver, lastTimestamp)
db.runInTransaction {
assets.open(indexAssetPath).use { indexStream ->
indexProcessor.process(indexStream)
@ -93,9 +93,9 @@ internal abstract class DbTest {
version: Long = 42L,
certificate: String = CERTIFICATE,
): Long {
val repoId = db.getRepositoryDao().insertEmptyRepo(address)
val repoId = db.getRepositoryDao().insertEmptyRepo(address, certificate = certificate)
val streamReceiver = DbV2StreamReceiver(db, repoId) { true }
val indexProcessor = IndexV2FullStreamProcessor(streamReceiver, certificate)
val indexProcessor = IndexV2FullStreamProcessor(streamReceiver)
db.runInTransaction {
assets.open(indexAssetPath).use { indexStream ->
indexProcessor.process(version, indexStream) {}

8
libs/database/src/dbTest/java/org/fdroid/database/IndexV1InsertTest.kt
View File

@ -62,7 +62,7 @@ internal class IndexV1InsertTest : DbTest() {
private fun streamIndex(path: String): Long {
val repoId = db.getRepositoryDao().insertEmptyRepo("https://f-droid.org/repo")
val streamReceiver = TestStreamReceiver(repoId)
val indexProcessor = IndexV1StreamProcessor(streamReceiver, null, -1)
val indexProcessor = IndexV1StreamProcessor(streamReceiver, -1)
db.runInTransaction {
assets.open(path).use { indexStream ->
indexProcessor.process(indexStream)
@ -80,7 +80,7 @@ internal class IndexV1InsertTest : DbTest() {
val streamReceiver = TestStreamReceiver(repoId) {
if (cIn.byteCount > 0) throw SerializationException()
}
val indexProcessor = IndexV1StreamProcessor(streamReceiver, null, -1)
val indexProcessor = IndexV1StreamProcessor(streamReceiver, -1)
cIn.use { indexStream ->
indexProcessor.process(indexStream)
}
@ -100,8 +100,8 @@ internal class IndexV1InsertTest : DbTest() {
private val callback: () -> Unit = {},
) : IndexV1StreamReceiver {
private val streamReceiver = DbV1StreamReceiver(db, repoId) { true }
override fun receive(repo: RepoV2, version: Long, certificate: String?) {
streamReceiver.receive(repo, version, certificate)
override fun receive(repo: RepoV2, version: Long) {
streamReceiver.receive(repo, version)
callback()
}

2
libs/database/src/dbTest/java/org/fdroid/database/IndexV2InsertTest.kt
View File

@ -64,7 +64,7 @@ internal class IndexV2InsertTest : DbTest() {
db.runInTransaction {
val repoId = db.getRepositoryDao().insertEmptyRepo("http://example.org")
val streamReceiver = DbV2StreamReceiver(db, repoId, compatibilityChecker)
val indexProcessor = IndexV2FullStreamProcessor(streamReceiver, "")
val indexProcessor = IndexV2FullStreamProcessor(streamReceiver)
cIn.use { indexStream ->
indexProcessor.process(42, indexStream) {}
}

12
libs/database/src/dbTest/java/org/fdroid/database/MultiRepoMigrationTest.kt
View File

@ -232,6 +232,7 @@ internal class MultiRepoMigrationTest {
// now get the Room DB, so we can use our DAOs for verifying the migration
databaseBuilder(getApplicationContext(), FDroidDatabaseInt::class.java, TEST_DB)
.addMigrations(MIGRATION_2_3)
.allowMainThreadQueries()
.build()
.use { db ->
@ -274,15 +275,11 @@ internal class MultiRepoMigrationTest {
// now get the Room DB, so we can use our DAOs for verifying the migration
databaseBuilder(getApplicationContext(), FDroidDatabaseInt::class.java, TEST_DB)
.addMigrations(MIGRATION_2_3)
.allowMainThreadQueries()
.build().use { db ->
// repo without cert did not get migrated
assertEquals(1, db.getRepositoryDao().getRepositories().size)
val repo = db.getRepositoryDao().getRepositories()[0]
// cert is still null
assertNull(repo.certificate)
// address still the same
assertEquals(fdroidRepo.address, repo.address)
// repo without cert did not get migrated, because we auto-migrate to latest version
assertEquals(0, db.getRepositoryDao().getRepositories().size)
}
}
@ -314,6 +311,7 @@ internal class MultiRepoMigrationTest {
// now get the Room DB, so we can use our DAOs for verifying the migration
databaseBuilder(getApplicationContext(), FDroidDatabaseInt::class.java, TEST_DB)
.addMigrations(MIGRATION_2_3)
.allowMainThreadQueries()
.build().use { db ->
check(db)

87
libs/database/src/dbTest/java/org/fdroid/database/RepoCertNonNullMigrationTest.kt Normal file
View File

@ -0,0 +1,87 @@
package org.fdroid.database
import android.content.ContentValues
import android.database.sqlite.SQLiteDatabase
import androidx.room.Room
import androidx.room.testing.MigrationTestHelper
import androidx.sqlite.db.framework.FrameworkSQLiteOpenHelperFactory
import androidx.test.core.app.ApplicationProvider.getApplicationContext
import androidx.test.ext.junit.runners.AndroidJUnit4
import androidx.test.platform.app.InstrumentationRegistry.getInstrumentation
import org.fdroid.database.Converters.localizedTextV2toString
import org.junit.Rule
import org.junit.Test
import org.junit.runner.RunWith
import kotlin.test.assertEquals
private const val TEST_DB = "migration-test"
@RunWith(AndroidJUnit4::class)
internal class RepoCertNonNullMigrationTest {
@get:Rule
val helper: MigrationTestHelper = MigrationTestHelper(
instrumentation = getInstrumentation(),
databaseClass = FDroidDatabaseInt::class.java,
specs = emptyList(),
openFactory = FrameworkSQLiteOpenHelperFactory(),
)
@Test
fun migrateRepos() {
helper.createDatabase(TEST_DB, 2).use { db ->
// Database has schema version 2. Insert some data using SQL queries.
// We can't use DAO classes because they expect the latest schema.
val repoId1 = db.insert(
CoreRepository.TABLE,
SQLiteDatabase.CONFLICT_FAIL,
ContentValues().apply {
put("name", localizedTextV2toString(mapOf("en-US" to "foo")))
put("description", localizedTextV2toString(mapOf("en-US" to "bar")))
put("address", "https://example.org/repo")
put("certificate", "0123")
put("timestamp", -1)
})
db.insert(
RepositoryPreferences.TABLE,
SQLiteDatabase.CONFLICT_FAIL,
ContentValues().apply {
put("repoId", repoId1)
put("enabled", true)
put("weight", Long.MAX_VALUE)
})
val repoId2 = db.insert(
CoreRepository.TABLE,
SQLiteDatabase.CONFLICT_FAIL,
ContentValues().apply {
put("name", localizedTextV2toString(mapOf("en-US" to "no cert")))
put("description", localizedTextV2toString(mapOf("en-US" to "no cert desc")))
put("address", "https://example.com/repo")
put("timestamp", -1)
})
db.insert(
RepositoryPreferences.TABLE,
SQLiteDatabase.CONFLICT_FAIL,
ContentValues().apply {
put("repoId", repoId2)
put("enabled", true)
put("weight", Long.MAX_VALUE - 2)
})
}
// Re-open the database with version 2, auto-migrations are applied automatically
helper.runMigrationsAndValidate(TEST_DB, 4, true, MIGRATION_2_3).close()
// now get the Room DB, so we can use our DAOs for verifying the migration
Room.databaseBuilder(getApplicationContext(), FDroidDatabaseInt::class.java, TEST_DB)
.addMigrations(MIGRATION_2_3)
.allowMainThreadQueries()
.build().use { db ->
// repo without cert did not get migrated, the other one did
assertEquals(1, db.getRepositoryDao().getRepositories().size)
val repo = db.getRepositoryDao().getRepositories()[0]
assertEquals("https://example.org/repo", repo.address)
}
}
}

17
libs/database/src/dbTest/java/org/fdroid/database/RepositoryDaoTest.kt
View File

@ -265,19 +265,6 @@ internal class RepositoryDaoTest : DbTest() {
assertEquals(repositoryPreferences, repoDao.getRepositoryPreferences(repoId))
}
@Test
fun certGetsUpdated() {
val repoId = repoDao.insertOrReplace(getRandomRepo())
assertEquals(1, repoDao.getRepositories().size)
assertEquals(null, repoDao.getRepositories()[0].certificate)
val cert = getRandomString()
repoDao.updateRepository(repoId, cert)
assertEquals(1, repoDao.getRepositories().size)
assertEquals(cert, repoDao.getRepositories()[0].certificate)
}
@Test
fun testGetMinRepositoryWeight() {
assertEquals(Int.MAX_VALUE, repoDao.getMinRepositoryWeight())
@ -344,13 +331,13 @@ internal class RepositoryDaoTest : DbTest() {
)
// we'll add an archive repo for repo1 to the list [3, 5, (1, 1a), 4, 2]
repoDao.updateRepository(repoId1, "1234abcd")
val repo1 = repoDao.getRepository(repoId1) ?: fail()
repoDao.updateRepository(repo1.repository.copy(certificate = "1234abcd"))
val repo1a = InitialRepository(
name = getRandomString(),
address = "https://example.org/archive",
description = getRandomString(),
certificate = repo1.certificate ?: fail(),
certificate = "1234abcd", // same as repo1
version = 42L,
enabled = false,
)

2
libs/database/src/dbTest/java/org/fdroid/index/IndexUpdaterTest.kt
View File

@ -17,7 +17,7 @@ internal class IndexUpdaterTest {
address = "http://example.org/",
timestamp = 1337L,
formatVersion = IndexFormatVersion.TWO,
certificate = null,
certificate = "abcd",
version = 2001,
weight = 0,
lastUpdated = 23L,

10
libs/database/src/dbTest/java/org/fdroid/index/v1/IndexV1UpdaterTest.kt
View File

@ -57,10 +57,10 @@ internal class IndexV1UpdaterTest : DbTest() {
@Test
fun testIndexV1Processing() {
val repoId = repoDao.insertEmptyRepo(TESTY_CANONICAL_URL)
val repoId = repoDao.insertEmptyRepo(TESTY_CANONICAL_URL, certificate = TESTY_CERT)
val repo = repoDao.getRepository(repoId) ?: fail()
downloadIndex(repo, TESTY_JAR)
val result = indexUpdater.updateNewRepo(repo, TESTY_FINGERPRINT).noError()
val result = indexUpdater.update(repo).noError()
assertIs<IndexUpdateResult.Processed>(result)
// repo got updated
@ -124,7 +124,7 @@ internal class IndexV1UpdaterTest : DbTest() {
val repoId = repoDao.insertEmptyRepo(TESTY_CANONICAL_URL)
val repo = repoDao.getRepository(repoId) ?: fail()
downloadIndex(repo, TESTY_JAR)
val result = indexUpdater.updateNewRepo(repo, "not the right fingerprint")
val result = indexUpdater.update(repo)
assertIs<IndexUpdateResult.Error>(result)
assertIs<SigningException>(result.e)
@ -141,7 +141,7 @@ internal class IndexV1UpdaterTest : DbTest() {
val futureRepo =
repo.copy(repository = repo.repository.copy(timestamp = System.currentTimeMillis()))
downloadIndex(futureRepo, TESTY_JAR)
val result = indexUpdater.updateNewRepo(futureRepo, TESTY_FINGERPRINT)
val result = indexUpdater.update(futureRepo)
assertIs<IndexUpdateResult.Error>(result)
assertIs<OldIndexException>(result.e)
assertFalse((result.e as OldIndexException).isSameTimestamp)
@ -208,7 +208,7 @@ internal class IndexV1UpdaterTest : DbTest() {
val repoId = repoDao.insertEmptyRepo("http://example.org")
val repo = repoDao.getRepository(repoId) ?: fail()
downloadIndex(repo, jar)
return indexUpdater.updateNewRepo(repo, null)
return indexUpdater.update(repo)
}
/**

40
libs/database/src/dbTest/java/org/fdroid/index/v2/IndexV2UpdaterTest.kt
View File

@ -21,15 +21,14 @@ import org.fdroid.test.TestDataMaxV2
import org.fdroid.test.TestDataMidV2
import org.fdroid.test.TestDataMinV2
import org.fdroid.test.VerifierConstants.CERTIFICATE
import org.fdroid.test.VerifierConstants.FINGERPRINT
import org.junit.Before
import org.junit.Rule
import org.junit.Test
import org.junit.rules.TemporaryFolder
import org.junit.runner.RunWith
import kotlin.test.assertEquals
import kotlin.test.assertIs
import kotlin.test.assertNull
import kotlin.test.assertTrue
import kotlin.test.fail
@RunWith(AndroidJUnit4::class)
@ -57,18 +56,18 @@ internal class IndexV2UpdaterTest : DbTest() {
@Test
fun testFullIndexEmptyToMin() {
val repoId = repoDao.insertEmptyRepo("http://example.org")
val repoId = repoDao.insertEmptyRepo("http://example.org", certificate = CERTIFICATE)
val repo = prepareUpdate(
repoId = repoId,
entryPath = "diff-empty-min/$SIGNED_FILE_NAME",
jsonPath = "index-min-v2.json",
indexFileV2 = TestDataEntry.emptyToMin.index
)
val result = indexUpdater.updateNewRepo(repo, FINGERPRINT).noError()
val result = indexUpdater.update(repo).noError()
assertEquals(IndexUpdateResult.Processed, result)
assertDbEquals(repoId, TestDataMinV2.index)
// check that certificate and format version got entered
// check that format version got entered and certificate stayed the same
val updatedRepo = repoDao.getRepository(repoId) ?: fail()
assertEquals(TWO, updatedRepo.formatVersion)
assertEquals(CERTIFICATE, updatedRepo.certificate)
@ -77,14 +76,14 @@ internal class IndexV2UpdaterTest : DbTest() {
@Test
fun testFullIndexEmptyToMid() {
val repoId = repoDao.insertEmptyRepo("http://example.org")
val repoId = repoDao.insertEmptyRepo("http://example.org", certificate = CERTIFICATE)
val repo = prepareUpdate(
repoId = repoId,
entryPath = "diff-empty-mid/$SIGNED_FILE_NAME",
jsonPath = "index-mid-v2.json",
indexFileV2 = TestDataEntry.emptyToMid.index
)
val result = indexUpdater.updateNewRepo(repo, FINGERPRINT).noError()
val result = indexUpdater.update(repo).noError()
assertEquals(IndexUpdateResult.Processed, result)
assertDbEquals(repoId, TestDataMidV2.index)
assertTimestampRecent(repoDao.getRepository(repoId)?.lastUpdated)
@ -92,14 +91,14 @@ internal class IndexV2UpdaterTest : DbTest() {
@Test
fun testFullIndexEmptyToMax() {
val repoId = repoDao.insertEmptyRepo("http://example.org")
val repoId = repoDao.insertEmptyRepo("http://example.org", certificate = CERTIFICATE)
val repo = prepareUpdate(
repoId = repoId,
entryPath = "diff-empty-max/$SIGNED_FILE_NAME",
jsonPath = "index-max-v2.json",
indexFileV2 = TestDataEntry.emptyToMax.index
)
val result = indexUpdater.updateNewRepo(repo, FINGERPRINT).noError()
val result = indexUpdater.update(repo).noError()
assertEquals(IndexUpdateResult.Processed, result)
assertDbEquals(repoId, TestDataMaxV2.index)
assertTimestampRecent(repoDao.getRepository(repoId)?.lastUpdated)
@ -123,7 +122,6 @@ internal class IndexV2UpdaterTest : DbTest() {
@Test
fun testDiffEmptyToMin() {
val repoId = streamIndexV2IntoDb("index-empty-v2.json")
repoDao.updateRepository(repoId, CERTIFICATE)
val repo = prepareUpdate(
repoId = repoId,
entryPath = "diff-empty-min/$SIGNED_FILE_NAME",
@ -139,7 +137,6 @@ internal class IndexV2UpdaterTest : DbTest() {
@Test
fun testDiffMidToMax() {
val repoId = streamIndexV2IntoDb("index-mid-v2.json")
repoDao.updateRepository(repoId, CERTIFICATE)
val repo = prepareUpdate(
repoId = repoId,
entryPath = "diff-empty-max/$SIGNED_FILE_NAME",
@ -155,7 +152,6 @@ internal class IndexV2UpdaterTest : DbTest() {
@Test
fun testSameTimestampUnchanged() {
val repoId = streamIndexV2IntoDb("index-min-v2.json")
repoDao.updateRepository(repoId, CERTIFICATE)
val repo = prepareUpdate(
repoId = repoId,
entryPath = "diff-empty-min/$SIGNED_FILE_NAME",
@ -171,7 +167,6 @@ internal class IndexV2UpdaterTest : DbTest() {
@Test
fun testHigherTimestampUnchanged() {
val repoId = streamIndexV2IntoDb("index-mid-v2.json")
repoDao.updateRepository(repoId, CERTIFICATE)
val repo = prepareUpdate(
repoId = repoId,
entryPath = "diff-empty-min/$SIGNED_FILE_NAME",
@ -186,7 +181,6 @@ internal class IndexV2UpdaterTest : DbTest() {
@Test
fun testNoDiffFoundIndexFallback() {
val repoId = streamIndexV2IntoDb("index-empty-v2.json")
repoDao.updateRepository(repoId, CERTIFICATE)
// fake timestamp of internal repo, so we will fail to find a diff in entry.json
val newRepo = repoDao.getRepository(repoId)?.repository?.copy(timestamp = 22) ?: fail()
repoDao.updateRepository(newRepo)
@ -203,20 +197,6 @@ internal class IndexV2UpdaterTest : DbTest() {
@Test
fun testWrongFingerprint() {
val repoId = repoDao.insertEmptyRepo("http://example.org")
val repo = prepareUpdate(
repoId = repoId,
entryPath = "diff-empty-min/$SIGNED_FILE_NAME",
jsonPath = "index-min-v2.json",
indexFileV2 = TestDataEntry.emptyToMin.index
)
val result = indexUpdater.updateNewRepo(repo, "wrong fingerprint")
assertTrue(result is IndexUpdateResult.Error)
assertTrue(result.e is SigningException)
}
@Test
fun testNormalUpdateOnRepoWithMissingFingerprint() {
val repoId = repoDao.insertEmptyRepo("http://example.org")
val repo = prepareUpdate(
repoId = repoId,
@ -225,8 +205,8 @@ internal class IndexV2UpdaterTest : DbTest() {
indexFileV2 = TestDataEntry.emptyToMin.index
)
val result = indexUpdater.update(repo)
assertTrue(result is IndexUpdateResult.Error)
assertTrue(result.e is IllegalArgumentException)
assertIs<IndexUpdateResult.Error>(result)
assertIs<SigningException>(result.e)
}
/**

4
libs/database/src/main/java/org/fdroid/database/DbV1StreamReceiver.kt
View File

@ -26,9 +26,9 @@ internal class DbV1StreamReceiver(
private val locales: LocaleListCompat = getLocales(Resources.getSystem().configuration)
override fun receive(repo: RepoV2, version: Long, certificate: String?) {
override fun receive(repo: RepoV2, version: Long) {
db.getRepositoryDao().clear(repoId)
db.getRepositoryDao().update(repoId, repo, version, ONE, certificate)
db.getRepositoryDao().update(repoId, repo, version, ONE)
}
override fun receive(packageName: String, m: MetadataV2) {

4
libs/database/src/main/java/org/fdroid/database/DbV2StreamReceiver.kt
View File

@ -36,10 +36,10 @@ internal class DbV2StreamReceiver(
}
@Synchronized
override fun receive(repo: RepoV2, version: Long, certificate: String) {
override fun receive(repo: RepoV2, version: Long) {
repo.walkFiles(nonNullFileV2)
clearRepoDataIfNeeded()
db.getRepositoryDao().update(repoId, repo, version, TWO, certificate)
db.getRepositoryDao().update(repoId, repo, version, TWO)
}
@Synchronized

4
libs/database/src/main/java/org/fdroid/database/FDroidDatabase.kt
View File

@ -16,7 +16,7 @@ import java.util.concurrent.Callable
// When bumping this version, please make sure to add one (or more) migration(s) below!
// Consider also providing tests for that migration.
// Don't forget to commit the new schema to the git repo as well.
version = 2,
version = 4,
entities = [
// repo
CoreRepository::class,
@ -44,6 +44,8 @@ import java.util.concurrent.Callable
exportSchema = true,
autoMigrations = [
AutoMigration(1, 2, MultiRepoMigration::class),
// 2 to 3 is a manual migration
AutoMigration(3, 4),
// add future migrations here (if they are easy enough to be done automatically)
],
)

1
libs/database/src/main/java/org/fdroid/database/FDroidDatabaseHolder.kt
View File

@ -58,6 +58,7 @@ public object FDroidDatabaseHolder {
FDroidDatabaseInt::class.java,
name,
).apply {
addMigrations(MIGRATION_2_3)
// We allow destructive migration (if no real migration was provided),
// so we have the option to nuke the DB in production (if that will ever be needed).
fallbackToDestructiveMigration()

11
libs/database/src/main/java/org/fdroid/database/Migrations.kt
View File

@ -4,6 +4,7 @@ import android.content.ContentValues
import android.database.Cursor
import android.database.sqlite.SQLiteDatabase.CONFLICT_FAIL
import androidx.room.migration.AutoMigrationSpec
import androidx.room.migration.Migration
import androidx.sqlite.db.SupportSQLiteDatabase
import mu.KotlinLogging
@ -104,3 +105,13 @@ internal class MultiRepoMigration : AutoMigrationSpec {
fun isArchive(): Boolean = address.trimEnd('/').endsWith("/archive")
}
}
/**
* Removes all repos without a certificate as those are broken anyway
* and force us to handle repos without certs.
*/
internal val MIGRATION_2_3 = object : Migration(2, 3) {
override fun migrate(db: SupportSQLiteDatabase) {
db.delete(CoreRepository.TABLE, "certificate IS NULL", null)
}
}

8
libs/database/src/main/java/org/fdroid/database/Repository.kt
View File

@ -31,7 +31,7 @@ internal data class CoreRepository(
val formatVersion: IndexFormatVersion?,
val maxAge: Int?,
val description: LocalizedTextV2 = emptyMap(),
val certificate: String?,
val certificate: String,
) {
internal companion object {
const val TABLE = "CoreRepository"
@ -47,7 +47,7 @@ internal fun RepoV2.toCoreRepository(
repoId: Long = 0,
version: Long,
formatVersion: IndexFormatVersion? = null,
certificate: String? = null,
certificate: String,
) = CoreRepository(
repoId = repoId,
name = name,
@ -99,7 +99,7 @@ public data class Repository internal constructor(
address: String,
timestamp: Long,
formatVersion: IndexFormatVersion,
certificate: String?,
certificate: String,
version: Long,
weight: Int,
lastUpdated: Long,
@ -135,7 +135,7 @@ public data class Repository internal constructor(
public val timestamp: Long get() = repository.timestamp
public val version: Long get() = repository.version ?: 0
public val formatVersion: IndexFormatVersion? get() = repository.formatVersion
public val certificate: String? get() = repository.certificate
public val certificate: String get() = repository.certificate
/**
* True if this repository is an archive repo.

25
libs/database/src/main/java/org/fdroid/database/RepositoryDao.kt
View File

@ -165,11 +165,13 @@ internal interface RepositoryDaoInt : RepositoryDao {
}
@Transaction
@VisibleForTesting
@Deprecated("Use insert instead")
fun insertEmptyRepo(
address: String,
username: String? = null,
password: String? = null,
certificate: String = "6789" // just used for testing
): Long {
val repo = CoreRepository(
name = mapOf("en-US" to address),
@ -179,7 +181,7 @@ internal interface RepositoryDaoInt : RepositoryDao {
version = null,
formatVersion = null,
maxAge = null,
certificate = null,
certificate = certificate,
)
val repoId = insertOrReplace(repo)
val currentMinWeight = getMinRepositoryWeight()
@ -197,7 +199,12 @@ internal interface RepositoryDaoInt : RepositoryDao {
@Transaction
@VisibleForTesting
fun insertOrReplace(repository: RepoV2, version: Long = 0): Long {
val repoId = insertOrReplace(repository.toCoreRepository(version = version))
val repoId = insertOrReplace(
repository.toCoreRepository(
version = version,
certificate = "0123", // just for testing
)
)
val currentMinWeight = getMinRepositoryWeight()
val repositoryPreferences = RepositoryPreferences(repoId, currentMinWeight - 2)
insert(repositoryPreferences)
@ -258,9 +265,9 @@ internal interface RepositoryDaoInt : RepositoryDao {
repository: RepoV2,
version: Long,
formatVersion: IndexFormatVersion,
certificate: String?,
) {
update(repository.toCoreRepository(repoId, version, formatVersion, certificate))
val repo = getRepository(repoId) ?: error("Repo with id $repoId did not exist")
update(repository.toCoreRepository(repoId, version, formatVersion, repo.certificate))
insertRepoTables(repoId, repository)
}
@ -274,16 +281,6 @@ internal interface RepositoryDaoInt : RepositoryDao {
@Update
fun updateRepository(repo: CoreRepository): Int
/**
* Updates the certificate for the [Repository] with the given [repoId].
* This should be used for V1 index updating where we only get the full cert
* after reading the entire index file.
* V2 index should use [update] instead as there the certificate is known
* before reading full index.
*/
@Query("UPDATE ${CoreRepository.TABLE} SET certificate = :certificate WHERE repoId = :repoId")
fun updateRepository(repoId: Long, certificate: String)
@Update
fun updateRepositoryPreferences(preferences: RepositoryPreferences)

23
libs/database/src/main/java/org/fdroid/index/IndexUpdater.kt
View File

@ -56,24 +56,11 @@ public abstract class IndexUpdater {
*/
public abstract val formatVersion: IndexFormatVersion
/**
* Updates a new [repo] for the first time.
*/
public fun updateNewRepo(
repo: Repository,
expectedSigningFingerprint: String?,
): IndexUpdateResult = catchExceptions {
update(repo, null, expectedSigningFingerprint)
}
/**
* Updates an existing [repo] with a known [Repository.certificate].
*/
public fun update(
repo: Repository,
): IndexUpdateResult = catchExceptions {
require(repo.certificate != null) { "Repo ${repo.address} had no certificate" }
update(repo, repo.certificate, null)
public fun update(repo: Repository): IndexUpdateResult = catchExceptions {
updateRepo(repo)
}
private fun catchExceptions(block: () -> IndexUpdateResult): IndexUpdateResult {
@ -86,11 +73,7 @@ public abstract class IndexUpdater {
}
}
protected abstract fun update(
repo: Repository,
certificate: String?,
fingerprint: String?,
): IndexUpdateResult
protected abstract fun updateRepo(repo: Repository): IndexUpdateResult
}
internal fun Downloader.setIndexUpdateListener(

23
libs/database/src/main/java/org/fdroid/index/RepoUpdater.kt
View File

@ -51,29 +51,8 @@ public class RepoUpdater(
/**
* Updates the given [repo].
* If [Repository.certificate] is null,
* the repo is considered to be new this being the first update.
*/
public fun update(
repo: Repository,
fingerprint: String? = null,
): IndexUpdateResult {
return if (repo.certificate == null) {
// This is a new repo without a certificate
updateNewRepo(repo, fingerprint)
} else {
update(repo)
}
}
private fun updateNewRepo(
repo: Repository,
expectedSigningFingerprint: String?,
): IndexUpdateResult = update(repo) { updater ->
updater.updateNewRepo(repo, expectedSigningFingerprint)
}
private fun update(repo: Repository): IndexUpdateResult = update(repo) { updater ->
public fun update(repo: Repository): IndexUpdateResult = update(repo) { updater ->
updater.update(repo)
}

19
libs/database/src/main/java/org/fdroid/index/v1/IndexV1Updater.kt
View File

@ -35,11 +35,7 @@ public class IndexV1Updater(
public override val formatVersion: IndexFormatVersion = ONE
private val db: FDroidDatabaseInt = database as FDroidDatabaseInt
override fun update(
repo: Repository,
certificate: String?,
fingerprint: String?,
): IndexUpdateResult {
override fun updateRepo(repo: Repository): IndexUpdateResult {
// Normally, we shouldn't allow repository downgrades and assert the condition below.
// However, F-Droid is concerned that late v2 bugs will require users to downgrade to v1,
// as it happened already with the migration from v0 to v1.
@ -61,21 +57,16 @@ public class IndexV1Updater(
if (!downloader.hasChanged()) return IndexUpdateResult.Unchanged
val eTag = downloader.cacheTag
val verifier = IndexV1Verifier(file, certificate, fingerprint)
val verifier = IndexV1Verifier(file, repo.certificate, null)
db.runInTransaction {
val (cert, _) = verifier.getStreamAndVerify { inputStream ->
verifier.getStreamAndVerify { inputStream ->
listener?.onUpdateProgress(repo, 0, 0)
val streamReceiver = DbV1StreamReceiver(db, repo.repoId, compatibilityChecker)
val streamProcessor =
IndexV1StreamProcessor(streamReceiver, certificate, repo.timestamp)
val streamProcessor = IndexV1StreamProcessor(streamReceiver, repo.timestamp)
streamProcessor.process(inputStream)
}
// update certificate, if we didn't have any before
val repoDao = db.getRepositoryDao()
if (certificate == null) {
repoDao.updateRepository(repo.repoId, cert)
}
// update RepositoryPreferences with timestamp and ETag (for v1)
val repoDao = db.getRepositoryDao()
val updatedPrefs = repo.preferences.copy(
lastUpdated = System.currentTimeMillis(),
lastETag = eTag,

18
libs/database/src/main/java/org/fdroid/index/v2/IndexV2Updater.kt
View File

@ -34,12 +34,8 @@ public class IndexV2Updater(
public override val formatVersion: IndexFormatVersion = TWO
private val db: FDroidDatabaseInt = database as FDroidDatabaseInt
override fun update(
repo: Repository,
certificate: String?,
fingerprint: String?,
): IndexUpdateResult {
val (cert, entry) = getCertAndEntry(repo, certificate, fingerprint)
override fun updateRepo(repo: Repository): IndexUpdateResult {
val (_, entry) = getCertAndEntry(repo, repo.certificate)
// don't process repos that we already did process in the past
if (entry.timestamp <= repo.timestamp) return IndexUpdateResult.Unchanged
// get diff, if available
@ -47,7 +43,7 @@ public class IndexV2Updater(
return if (diff == null || repo.formatVersion == ONE) {
// no diff found (or this is upgrade from v1 repo), so do full index update
val streamReceiver = DbV2StreamReceiver(db, repo.repoId, compatibilityChecker)
val streamProcessor = IndexV2FullStreamProcessor(streamReceiver, cert)
val streamProcessor = IndexV2FullStreamProcessor(streamReceiver)
processStream(repo, entry.index, entry.version, streamProcessor)
} else {
// use available diff
@ -57,11 +53,7 @@ public class IndexV2Updater(
}
}
private fun getCertAndEntry(
repo: Repository,
certificate: String?,
fingerprint: String?,
): Pair<String, Entry> {
private fun getCertAndEntry(repo: Repository, certificate: String): Pair<String, Entry> {
val file = tempFileProvider.createTempFile()
val downloader = downloaderFactory.createWithTryFirstMirror(
repo = repo,
@ -73,7 +65,7 @@ public class IndexV2Updater(
}
try {
downloader.download()
val verifier = EntryVerifier(file, certificate, fingerprint)
val verifier = EntryVerifier(file, certificate, null)
return verifier.getStreamAndVerify { inputStream ->
IndexParser.parseEntry(inputStream)
}

2
libs/database/src/main/java/org/fdroid/repo/RepoAdder.kt
View File

@ -366,7 +366,7 @@ internal class RepoAdder(
address = uri.toString(),
timestamp = -1L,
formatVersion = indexFormatVersion,
certificate = null,
certificate = "This is fake and will be replaced by real cert before saving in DB.",
version = 0L,
weight = 0,
lastUpdated = -1L,

4
libs/database/src/main/java/org/fdroid/repo/RepoV2Fetcher.kt
View File

@ -56,8 +56,8 @@ internal class RepoV2Fetcher(
log.info { "Downloaded entry, now streaming index..." }
val streamReceiver = RepoV2StreamReceiver(receiver, repo.username, repo.password)
val streamProcessor = IndexV2FullStreamProcessor(streamReceiver, cert)
val streamReceiver = RepoV2StreamReceiver(receiver, cert, repo.username, repo.password)
val streamProcessor = IndexV2FullStreamProcessor(streamReceiver)
val digestInputStream = if (uri.scheme?.startsWith("http") == true) {
// stream index for http(s) downloads
val indexRequest = DownloadRequest(

5
libs/database/src/main/java/org/fdroid/repo/RepoV2StreamReceiver.kt
View File

@ -19,6 +19,7 @@ import org.fdroid.index.v2.RepoV2
internal open class RepoV2StreamReceiver(
private val receiver: RepoPreviewReceiver,
private val certificate: String,
private val username: String?,
private val password: String?,
) : IndexV2StreamReceiver {
@ -28,7 +29,7 @@ internal open class RepoV2StreamReceiver(
repo: RepoV2,
version: Long,
formatVersion: IndexFormatVersion,
certificate: String?,
certificate: String,
username: String?,
password: String?,
) = Repository(
@ -80,7 +81,7 @@ internal open class RepoV2StreamReceiver(
private val locales: LocaleListCompat = getLocales(Resources.getSystem().configuration)
override fun receive(repo: RepoV2, version: Long, certificate: String) {
override fun receive(repo: RepoV2, version: Long) {
receiver.onRepoReceived(
getRepository(
repo = repo,

8
libs/database/src/test/java/org/fdroid/database/DbV2StreamReceiverTest.kt
View File

@ -27,25 +27,25 @@ internal class DbV2StreamReceiverTest {
timestamp = 42L,
)
every { db.getRepositoryDao() } returns mockk(relaxed = true)
dbV2StreamReceiver.receive(repoV2, 42L, "cert")
dbV2StreamReceiver.receive(repoV2, 42L)
// icon file without leading / does not pass
val repoV2NoSlash =
repoV2.copy(icon = mapOf("en" to FileV2(name = "foo", sha256 = "bar", size = 23L)))
assertFailsWith<SerializationException> {
dbV2StreamReceiver.receive(repoV2NoSlash, 42L, "cert")
dbV2StreamReceiver.receive(repoV2NoSlash, 42L)
}
// icon file without sha256 hash fails
val repoNoSha256 = repoV2.copy(icon = mapOf("en" to FileV2(name = "/foo", size = 23L)))
assertFailsWith<SerializationException> {
dbV2StreamReceiver.receive(repoNoSha256, 42L, "cert")
dbV2StreamReceiver.receive(repoNoSha256, 42L)
}
// icon file without size fails
val repoNoSize = repoV2.copy(icon = mapOf("en" to FileV2(name = "/foo", sha256 = "bar")))
assertFailsWith<SerializationException> {
dbV2StreamReceiver.receive(repoNoSize, 42L, "cert")
dbV2StreamReceiver.receive(repoNoSize, 42L)
}
}
}