Merge branch 'remove-repos-no-cert' into 'master'

Remove broken repos without certificate and remove code handling that

See merge request fdroid/fdroidclient!1377
This commit is contained in:
Torsten Grote 2024-04-12 12:49:00 +00:00
commit 8eaaf2d25c
36 changed files with 2397 additions and 208 deletions

View File

@ -521,6 +521,7 @@ public class FDroidApp extends Application implements androidx.work.Configuratio
public static Repository createSwapRepo(String address, String certificate) {
long now = System.currentTimeMillis();
if (certificate == null) certificate = "d0ef";
return new Repository(42L, address, now, IndexFormatVersion.ONE, certificate, 20001L, 42,
now);
}

View File

@ -35,7 +35,6 @@ import android.util.Log;
import android.widget.Toast;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.annotation.UiThread;
import androidx.core.app.JobIntentService;
import androidx.core.app.NotificationCompat;
@ -78,7 +77,6 @@ public class UpdateService extends JobIntentService {
public static final String LOCAL_ACTION_STATUS = "status";
private static final String EXTRA_MESSAGE = "msg";
private static final String EXTRA_REPO_FINGERPRINT = "fingerprint";
private static final String EXTRA_REPO_ERRORS = "repoErrors";
public static final String EXTRA_STATUS_CODE = "status";
private static final String EXTRA_MANUAL_UPDATE = "manualUpdate";
@ -115,14 +113,9 @@ public class UpdateService extends JobIntentService {
updateRepoNow(context, null);
}
public static void updateRepoNow(Context context, String address) {
updateNewRepoNow(context, address, null);
}
public static Intent getIntent(Context context, String address, @Nullable String fingerprint) {
public static Intent getIntent(Context context, String address) {
Intent intent = new Intent(context, UpdateService.class);
intent.putExtra(EXTRA_MANUAL_UPDATE, true);
intent.putExtra(EXTRA_REPO_FINGERPRINT, fingerprint);
if (!TextUtils.isEmpty(address)) {
intent.setData(Uri.parse(address));
}
@ -130,8 +123,8 @@ public class UpdateService extends JobIntentService {
}
@UiThread
public static void updateNewRepoNow(Context context, String address, @Nullable String fingerprint) {
enqueueWork(context, getIntent(context, address, fingerprint));
public static void updateRepoNow(Context context, String address) {
enqueueWork(context, getIntent(context, address));
}
/**
@ -396,7 +389,6 @@ public class UpdateService extends JobIntentService {
boolean manualUpdate = intent.getBooleanExtra(EXTRA_MANUAL_UPDATE, false);
boolean forcedUpdate = intent.getBooleanExtra(EXTRA_FORCED_UPDATE, false);
isForcedUpdate = forcedUpdate;
String fingerprint = intent.getStringExtra(EXTRA_REPO_FINGERPRINT);
String address = intent.getDataString();
try {
@ -465,11 +457,11 @@ public class UpdateService extends JobIntentService {
File.createTempFile("dl-", "", cacheDir);
final IndexV1Updater updater = new IndexV1Updater(db, tempFileProvider,
DownloaderFactory.INSTANCE, repoUriBuilder, compatChecker, listener);
result = updater.updateNewRepo(repo, fingerprint);
result = updater.update(repo);
} else {
final RepoUpdater updater = new RepoUpdater(cacheDir, db,
DownloaderFactory.INSTANCE, repoUriBuilder, compatChecker, listener);
result = updater.update(repo, fingerprint);
result = updater.update(repo);
}
if (result instanceof IndexUpdateResult.Unchanged) {
unchangedRepos++;

View File

@ -193,7 +193,7 @@ private fun getRepoString(repo: Repository, isPreferred: Boolean) = buildAnnotat
@Composable
@Preview(uiMode = Configuration.UI_MODE_NIGHT_NO)
fun RepoChooserSingleRepoPreview() {
val repo1 = Repository(1L, "1", 1L, TWO, null, 1L, 1, 1L)
val repo1 = Repository(1L, "1", 1L, TWO, "null", 1L, 1, 1L)
FDroidContent {
RepoChooser(listOf(repo1), 1L, 1L, {}, {})
}
@ -202,9 +202,9 @@ fun RepoChooserSingleRepoPreview() {
@Composable
@Preview(uiMode = Configuration.UI_MODE_NIGHT_NO)
fun RepoChooserPreview() {
val repo1 = Repository(1L, "1", 1L, TWO, null, 1L, 1, 1L)
val repo2 = Repository(2L, "2", 2L, TWO, null, 2L, 2, 2L)
val repo3 = Repository(3L, "2", 3L, TWO, null, 3L, 3, 3L)
val repo1 = Repository(1L, "1", 1L, TWO, "null", 1L, 1, 1L)
val repo2 = Repository(2L, "2", 2L, TWO, "null", 2L, 2, 2L)
val repo3 = Repository(3L, "2", 3L, TWO, "null", 3L, 3, 3L)
FDroidContent {
RepoChooser(listOf(repo1, repo2, repo3), 1L, 1L, {}, {})
}
@ -213,9 +213,9 @@ fun RepoChooserPreview() {
@Composable
@Preview(uiMode = Configuration.UI_MODE_NIGHT_YES)
fun RepoChooserNightPreview() {
val repo1 = Repository(1L, "1", 1L, TWO, null, 1L, 1, 1L)
val repo2 = Repository(2L, "2", 2L, TWO, null, 2L, 2, 2L)
val repo3 = Repository(3L, "2", 3L, TWO, null, 3L, 3, 3L)
val repo1 = Repository(1L, "1", 1L, TWO, "null", 1L, 1, 1L)
val repo2 = Repository(2L, "2", 2L, TWO, "null", 2L, 2, 2L)
val repo3 = Repository(3L, "2", 3L, TWO, "null", 3L, 3, 3L)
FDroidContent {
RepoChooser(listOf(repo1, repo2, repo3), 1L, 2L, {}, {})
}

View File

@ -143,7 +143,7 @@ public class UpdateServiceTest {
.putInt(Preferences.PREF_OVER_DATA, Preferences.OVER_NETWORK_ALWAYS)
.putInt(Preferences.PREF_OVER_WIFI, Preferences.OVER_NETWORK_ALWAYS)
.commit();
final Intent intent = UpdateService.getIntent(context, address, fingerprint);
final Intent intent = UpdateService.getIntent(context, address);
final TestUpdateService testUpdateService = Robolectric.buildService(TestUpdateService.class,
intent).bind().get();
Thread t = new Thread() {

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -76,9 +76,9 @@ internal abstract class DbTest {
certificate: String = CERTIFICATE,
lastTimestamp: Long = -1,
): Long {
val repoId = db.getRepositoryDao().insertEmptyRepo(address)
val repoId = db.getRepositoryDao().insertEmptyRepo(address, certificate = certificate)
val streamReceiver = DbV1StreamReceiver(db, repoId) { true }
val indexProcessor = IndexV1StreamProcessor(streamReceiver, certificate, lastTimestamp)
val indexProcessor = IndexV1StreamProcessor(streamReceiver, lastTimestamp)
db.runInTransaction {
assets.open(indexAssetPath).use { indexStream ->
indexProcessor.process(indexStream)
@ -93,9 +93,9 @@ internal abstract class DbTest {
version: Long = 42L,
certificate: String = CERTIFICATE,
): Long {
val repoId = db.getRepositoryDao().insertEmptyRepo(address)
val repoId = db.getRepositoryDao().insertEmptyRepo(address, certificate = certificate)
val streamReceiver = DbV2StreamReceiver(db, repoId) { true }
val indexProcessor = IndexV2FullStreamProcessor(streamReceiver, certificate)
val indexProcessor = IndexV2FullStreamProcessor(streamReceiver)
db.runInTransaction {
assets.open(indexAssetPath).use { indexStream ->
indexProcessor.process(version, indexStream) {}

View File

@ -62,7 +62,7 @@ internal class IndexV1InsertTest : DbTest() {
private fun streamIndex(path: String): Long {
val repoId = db.getRepositoryDao().insertEmptyRepo("https://f-droid.org/repo")
val streamReceiver = TestStreamReceiver(repoId)
val indexProcessor = IndexV1StreamProcessor(streamReceiver, null, -1)
val indexProcessor = IndexV1StreamProcessor(streamReceiver, -1)
db.runInTransaction {
assets.open(path).use { indexStream ->
indexProcessor.process(indexStream)
@ -80,7 +80,7 @@ internal class IndexV1InsertTest : DbTest() {
val streamReceiver = TestStreamReceiver(repoId) {
if (cIn.byteCount > 0) throw SerializationException()
}
val indexProcessor = IndexV1StreamProcessor(streamReceiver, null, -1)
val indexProcessor = IndexV1StreamProcessor(streamReceiver, -1)
cIn.use { indexStream ->
indexProcessor.process(indexStream)
}
@ -100,8 +100,8 @@ internal class IndexV1InsertTest : DbTest() {
private val callback: () -> Unit = {},
) : IndexV1StreamReceiver {
private val streamReceiver = DbV1StreamReceiver(db, repoId) { true }
override fun receive(repo: RepoV2, version: Long, certificate: String?) {
streamReceiver.receive(repo, version, certificate)
override fun receive(repo: RepoV2, version: Long) {
streamReceiver.receive(repo, version)
callback()
}

View File

@ -64,7 +64,7 @@ internal class IndexV2InsertTest : DbTest() {
db.runInTransaction {
val repoId = db.getRepositoryDao().insertEmptyRepo("http://example.org")
val streamReceiver = DbV2StreamReceiver(db, repoId, compatibilityChecker)
val indexProcessor = IndexV2FullStreamProcessor(streamReceiver, "")
val indexProcessor = IndexV2FullStreamProcessor(streamReceiver)
cIn.use { indexStream ->
indexProcessor.process(42, indexStream) {}
}

View File

@ -232,6 +232,7 @@ internal class MultiRepoMigrationTest {
// now get the Room DB, so we can use our DAOs for verifying the migration
databaseBuilder(getApplicationContext(), FDroidDatabaseInt::class.java, TEST_DB)
.addMigrations(MIGRATION_2_3)
.allowMainThreadQueries()
.build()
.use { db ->
@ -274,15 +275,11 @@ internal class MultiRepoMigrationTest {
// now get the Room DB, so we can use our DAOs for verifying the migration
databaseBuilder(getApplicationContext(), FDroidDatabaseInt::class.java, TEST_DB)
.addMigrations(MIGRATION_2_3)
.allowMainThreadQueries()
.build().use { db ->
// repo without cert did not get migrated
assertEquals(1, db.getRepositoryDao().getRepositories().size)
val repo = db.getRepositoryDao().getRepositories()[0]
// cert is still null
assertNull(repo.certificate)
// address still the same
assertEquals(fdroidRepo.address, repo.address)
// repo without cert did not get migrated, because we auto-migrate to latest version
assertEquals(0, db.getRepositoryDao().getRepositories().size)
}
}
@ -314,6 +311,7 @@ internal class MultiRepoMigrationTest {
// now get the Room DB, so we can use our DAOs for verifying the migration
databaseBuilder(getApplicationContext(), FDroidDatabaseInt::class.java, TEST_DB)
.addMigrations(MIGRATION_2_3)
.allowMainThreadQueries()
.build().use { db ->
check(db)

View File

@ -0,0 +1,87 @@
package org.fdroid.database
import android.content.ContentValues
import android.database.sqlite.SQLiteDatabase
import androidx.room.Room
import androidx.room.testing.MigrationTestHelper
import androidx.sqlite.db.framework.FrameworkSQLiteOpenHelperFactory
import androidx.test.core.app.ApplicationProvider.getApplicationContext
import androidx.test.ext.junit.runners.AndroidJUnit4
import androidx.test.platform.app.InstrumentationRegistry.getInstrumentation
import org.fdroid.database.Converters.localizedTextV2toString
import org.junit.Rule
import org.junit.Test
import org.junit.runner.RunWith
import kotlin.test.assertEquals
private const val TEST_DB = "migration-test"
@RunWith(AndroidJUnit4::class)
internal class RepoCertNonNullMigrationTest {
@get:Rule
val helper: MigrationTestHelper = MigrationTestHelper(
instrumentation = getInstrumentation(),
databaseClass = FDroidDatabaseInt::class.java,
specs = emptyList(),
openFactory = FrameworkSQLiteOpenHelperFactory(),
)
@Test
fun migrateRepos() {
helper.createDatabase(TEST_DB, 2).use { db ->
// Database has schema version 2. Insert some data using SQL queries.
// We can't use DAO classes because they expect the latest schema.
val repoId1 = db.insert(
CoreRepository.TABLE,
SQLiteDatabase.CONFLICT_FAIL,
ContentValues().apply {
put("name", localizedTextV2toString(mapOf("en-US" to "foo")))
put("description", localizedTextV2toString(mapOf("en-US" to "bar")))
put("address", "https://example.org/repo")
put("certificate", "0123")
put("timestamp", -1)
})
db.insert(
RepositoryPreferences.TABLE,
SQLiteDatabase.CONFLICT_FAIL,
ContentValues().apply {
put("repoId", repoId1)
put("enabled", true)
put("weight", Long.MAX_VALUE)
})
val repoId2 = db.insert(
CoreRepository.TABLE,
SQLiteDatabase.CONFLICT_FAIL,
ContentValues().apply {
put("name", localizedTextV2toString(mapOf("en-US" to "no cert")))
put("description", localizedTextV2toString(mapOf("en-US" to "no cert desc")))
put("address", "https://example.com/repo")
put("timestamp", -1)
})
db.insert(
RepositoryPreferences.TABLE,
SQLiteDatabase.CONFLICT_FAIL,
ContentValues().apply {
put("repoId", repoId2)
put("enabled", true)
put("weight", Long.MAX_VALUE - 2)
})
}
// Re-open the database with version 2, auto-migrations are applied automatically
helper.runMigrationsAndValidate(TEST_DB, 4, true, MIGRATION_2_3).close()
// now get the Room DB, so we can use our DAOs for verifying the migration
Room.databaseBuilder(getApplicationContext(), FDroidDatabaseInt::class.java, TEST_DB)
.addMigrations(MIGRATION_2_3)
.allowMainThreadQueries()
.build().use { db ->
// repo without cert did not get migrated, the other one did
assertEquals(1, db.getRepositoryDao().getRepositories().size)
val repo = db.getRepositoryDao().getRepositories()[0]
assertEquals("https://example.org/repo", repo.address)
}
}
}

View File

@ -265,19 +265,6 @@ internal class RepositoryDaoTest : DbTest() {
assertEquals(repositoryPreferences, repoDao.getRepositoryPreferences(repoId))
}
@Test
fun certGetsUpdated() {
val repoId = repoDao.insertOrReplace(getRandomRepo())
assertEquals(1, repoDao.getRepositories().size)
assertEquals(null, repoDao.getRepositories()[0].certificate)
val cert = getRandomString()
repoDao.updateRepository(repoId, cert)
assertEquals(1, repoDao.getRepositories().size)
assertEquals(cert, repoDao.getRepositories()[0].certificate)
}
@Test
fun testGetMinRepositoryWeight() {
assertEquals(Int.MAX_VALUE, repoDao.getMinRepositoryWeight())
@ -344,13 +331,13 @@ internal class RepositoryDaoTest : DbTest() {
)
// we'll add an archive repo for repo1 to the list [3, 5, (1, 1a), 4, 2]
repoDao.updateRepository(repoId1, "1234abcd")
val repo1 = repoDao.getRepository(repoId1) ?: fail()
repoDao.updateRepository(repo1.repository.copy(certificate = "1234abcd"))
val repo1a = InitialRepository(
name = getRandomString(),
address = "https://example.org/archive",
description = getRandomString(),
certificate = repo1.certificate ?: fail(),
certificate = "1234abcd", // same as repo1
version = 42L,
enabled = false,
)

View File

@ -17,7 +17,7 @@ internal class IndexUpdaterTest {
address = "http://example.org/",
timestamp = 1337L,
formatVersion = IndexFormatVersion.TWO,
certificate = null,
certificate = "abcd",
version = 2001,
weight = 0,
lastUpdated = 23L,

View File

@ -57,10 +57,10 @@ internal class IndexV1UpdaterTest : DbTest() {
@Test
fun testIndexV1Processing() {
val repoId = repoDao.insertEmptyRepo(TESTY_CANONICAL_URL)
val repoId = repoDao.insertEmptyRepo(TESTY_CANONICAL_URL, certificate = TESTY_CERT)
val repo = repoDao.getRepository(repoId) ?: fail()
downloadIndex(repo, TESTY_JAR)
val result = indexUpdater.updateNewRepo(repo, TESTY_FINGERPRINT).noError()
val result = indexUpdater.update(repo).noError()
assertIs<IndexUpdateResult.Processed>(result)
// repo got updated
@ -124,7 +124,7 @@ internal class IndexV1UpdaterTest : DbTest() {
val repoId = repoDao.insertEmptyRepo(TESTY_CANONICAL_URL)
val repo = repoDao.getRepository(repoId) ?: fail()
downloadIndex(repo, TESTY_JAR)
val result = indexUpdater.updateNewRepo(repo, "not the right fingerprint")
val result = indexUpdater.update(repo)
assertIs<IndexUpdateResult.Error>(result)
assertIs<SigningException>(result.e)
@ -141,7 +141,7 @@ internal class IndexV1UpdaterTest : DbTest() {
val futureRepo =
repo.copy(repository = repo.repository.copy(timestamp = System.currentTimeMillis()))
downloadIndex(futureRepo, TESTY_JAR)
val result = indexUpdater.updateNewRepo(futureRepo, TESTY_FINGERPRINT)
val result = indexUpdater.update(futureRepo)
assertIs<IndexUpdateResult.Error>(result)
assertIs<OldIndexException>(result.e)
assertFalse((result.e as OldIndexException).isSameTimestamp)
@ -208,7 +208,7 @@ internal class IndexV1UpdaterTest : DbTest() {
val repoId = repoDao.insertEmptyRepo("http://example.org")
val repo = repoDao.getRepository(repoId) ?: fail()
downloadIndex(repo, jar)
return indexUpdater.updateNewRepo(repo, null)
return indexUpdater.update(repo)
}
/**

View File

@ -21,15 +21,14 @@ import org.fdroid.test.TestDataMaxV2
import org.fdroid.test.TestDataMidV2
import org.fdroid.test.TestDataMinV2
import org.fdroid.test.VerifierConstants.CERTIFICATE
import org.fdroid.test.VerifierConstants.FINGERPRINT
import org.junit.Before
import org.junit.Rule
import org.junit.Test
import org.junit.rules.TemporaryFolder
import org.junit.runner.RunWith
import kotlin.test.assertEquals
import kotlin.test.assertIs
import kotlin.test.assertNull
import kotlin.test.assertTrue
import kotlin.test.fail
@RunWith(AndroidJUnit4::class)
@ -57,18 +56,18 @@ internal class IndexV2UpdaterTest : DbTest() {
@Test
fun testFullIndexEmptyToMin() {
val repoId = repoDao.insertEmptyRepo("http://example.org")
val repoId = repoDao.insertEmptyRepo("http://example.org", certificate = CERTIFICATE)
val repo = prepareUpdate(
repoId = repoId,
entryPath = "diff-empty-min/$SIGNED_FILE_NAME",
jsonPath = "index-min-v2.json",
indexFileV2 = TestDataEntry.emptyToMin.index
)
val result = indexUpdater.updateNewRepo(repo, FINGERPRINT).noError()
val result = indexUpdater.update(repo).noError()
assertEquals(IndexUpdateResult.Processed, result)
assertDbEquals(repoId, TestDataMinV2.index)
// check that certificate and format version got entered
// check that format version got entered and certificate stayed the same
val updatedRepo = repoDao.getRepository(repoId) ?: fail()
assertEquals(TWO, updatedRepo.formatVersion)
assertEquals(CERTIFICATE, updatedRepo.certificate)
@ -77,14 +76,14 @@ internal class IndexV2UpdaterTest : DbTest() {
@Test
fun testFullIndexEmptyToMid() {
val repoId = repoDao.insertEmptyRepo("http://example.org")
val repoId = repoDao.insertEmptyRepo("http://example.org", certificate = CERTIFICATE)
val repo = prepareUpdate(
repoId = repoId,
entryPath = "diff-empty-mid/$SIGNED_FILE_NAME",
jsonPath = "index-mid-v2.json",
indexFileV2 = TestDataEntry.emptyToMid.index
)
val result = indexUpdater.updateNewRepo(repo, FINGERPRINT).noError()
val result = indexUpdater.update(repo).noError()
assertEquals(IndexUpdateResult.Processed, result)
assertDbEquals(repoId, TestDataMidV2.index)
assertTimestampRecent(repoDao.getRepository(repoId)?.lastUpdated)
@ -92,14 +91,14 @@ internal class IndexV2UpdaterTest : DbTest() {
@Test
fun testFullIndexEmptyToMax() {
val repoId = repoDao.insertEmptyRepo("http://example.org")
val repoId = repoDao.insertEmptyRepo("http://example.org", certificate = CERTIFICATE)
val repo = prepareUpdate(
repoId = repoId,
entryPath = "diff-empty-max/$SIGNED_FILE_NAME",
jsonPath = "index-max-v2.json",
indexFileV2 = TestDataEntry.emptyToMax.index
)
val result = indexUpdater.updateNewRepo(repo, FINGERPRINT).noError()
val result = indexUpdater.update(repo).noError()
assertEquals(IndexUpdateResult.Processed, result)
assertDbEquals(repoId, TestDataMaxV2.index)
assertTimestampRecent(repoDao.getRepository(repoId)?.lastUpdated)
@ -123,7 +122,6 @@ internal class IndexV2UpdaterTest : DbTest() {
@Test
fun testDiffEmptyToMin() {
val repoId = streamIndexV2IntoDb("index-empty-v2.json")
repoDao.updateRepository(repoId, CERTIFICATE)
val repo = prepareUpdate(
repoId = repoId,
entryPath = "diff-empty-min/$SIGNED_FILE_NAME",
@ -139,7 +137,6 @@ internal class IndexV2UpdaterTest : DbTest() {
@Test
fun testDiffMidToMax() {
val repoId = streamIndexV2IntoDb("index-mid-v2.json")
repoDao.updateRepository(repoId, CERTIFICATE)
val repo = prepareUpdate(
repoId = repoId,
entryPath = "diff-empty-max/$SIGNED_FILE_NAME",
@ -155,7 +152,6 @@ internal class IndexV2UpdaterTest : DbTest() {
@Test
fun testSameTimestampUnchanged() {
val repoId = streamIndexV2IntoDb("index-min-v2.json")
repoDao.updateRepository(repoId, CERTIFICATE)
val repo = prepareUpdate(
repoId = repoId,
entryPath = "diff-empty-min/$SIGNED_FILE_NAME",
@ -171,7 +167,6 @@ internal class IndexV2UpdaterTest : DbTest() {
@Test
fun testHigherTimestampUnchanged() {
val repoId = streamIndexV2IntoDb("index-mid-v2.json")
repoDao.updateRepository(repoId, CERTIFICATE)
val repo = prepareUpdate(
repoId = repoId,
entryPath = "diff-empty-min/$SIGNED_FILE_NAME",
@ -186,7 +181,6 @@ internal class IndexV2UpdaterTest : DbTest() {
@Test
fun testNoDiffFoundIndexFallback() {
val repoId = streamIndexV2IntoDb("index-empty-v2.json")
repoDao.updateRepository(repoId, CERTIFICATE)
// fake timestamp of internal repo, so we will fail to find a diff in entry.json
val newRepo = repoDao.getRepository(repoId)?.repository?.copy(timestamp = 22) ?: fail()
repoDao.updateRepository(newRepo)
@ -203,20 +197,6 @@ internal class IndexV2UpdaterTest : DbTest() {
@Test
fun testWrongFingerprint() {
val repoId = repoDao.insertEmptyRepo("http://example.org")
val repo = prepareUpdate(
repoId = repoId,
entryPath = "diff-empty-min/$SIGNED_FILE_NAME",
jsonPath = "index-min-v2.json",
indexFileV2 = TestDataEntry.emptyToMin.index
)
val result = indexUpdater.updateNewRepo(repo, "wrong fingerprint")
assertTrue(result is IndexUpdateResult.Error)
assertTrue(result.e is SigningException)
}
@Test
fun testNormalUpdateOnRepoWithMissingFingerprint() {
val repoId = repoDao.insertEmptyRepo("http://example.org")
val repo = prepareUpdate(
repoId = repoId,
@ -225,8 +205,8 @@ internal class IndexV2UpdaterTest : DbTest() {
indexFileV2 = TestDataEntry.emptyToMin.index
)
val result = indexUpdater.update(repo)
assertTrue(result is IndexUpdateResult.Error)
assertTrue(result.e is IllegalArgumentException)
assertIs<IndexUpdateResult.Error>(result)
assertIs<SigningException>(result.e)
}
/**

View File

@ -26,9 +26,9 @@ internal class DbV1StreamReceiver(
private val locales: LocaleListCompat = getLocales(Resources.getSystem().configuration)
override fun receive(repo: RepoV2, version: Long, certificate: String?) {
override fun receive(repo: RepoV2, version: Long) {
db.getRepositoryDao().clear(repoId)
db.getRepositoryDao().update(repoId, repo, version, ONE, certificate)
db.getRepositoryDao().update(repoId, repo, version, ONE)
}
override fun receive(packageName: String, m: MetadataV2) {

View File

@ -36,10 +36,10 @@ internal class DbV2StreamReceiver(
}
@Synchronized
override fun receive(repo: RepoV2, version: Long, certificate: String) {
override fun receive(repo: RepoV2, version: Long) {
repo.walkFiles(nonNullFileV2)
clearRepoDataIfNeeded()
db.getRepositoryDao().update(repoId, repo, version, TWO, certificate)
db.getRepositoryDao().update(repoId, repo, version, TWO)
}
@Synchronized

View File

@ -16,7 +16,7 @@ import java.util.concurrent.Callable
// When bumping this version, please make sure to add one (or more) migration(s) below!
// Consider also providing tests for that migration.
// Don't forget to commit the new schema to the git repo as well.
version = 2,
version = 4,
entities = [
// repo
CoreRepository::class,
@ -44,6 +44,8 @@ import java.util.concurrent.Callable
exportSchema = true,
autoMigrations = [
AutoMigration(1, 2, MultiRepoMigration::class),
// 2 to 3 is a manual migration
AutoMigration(3, 4),
// add future migrations here (if they are easy enough to be done automatically)
],
)

View File

@ -58,6 +58,7 @@ public object FDroidDatabaseHolder {
FDroidDatabaseInt::class.java,
name,
).apply {
addMigrations(MIGRATION_2_3)
// We allow destructive migration (if no real migration was provided),
// so we have the option to nuke the DB in production (if that will ever be needed).
fallbackToDestructiveMigration()

View File

@ -4,6 +4,7 @@ import android.content.ContentValues
import android.database.Cursor
import android.database.sqlite.SQLiteDatabase.CONFLICT_FAIL
import androidx.room.migration.AutoMigrationSpec
import androidx.room.migration.Migration
import androidx.sqlite.db.SupportSQLiteDatabase
import mu.KotlinLogging
@ -104,3 +105,13 @@ internal class MultiRepoMigration : AutoMigrationSpec {
fun isArchive(): Boolean = address.trimEnd('/').endsWith("/archive")
}
}
/**
* Removes all repos without a certificate as those are broken anyway
* and force us to handle repos without certs.
*/
internal val MIGRATION_2_3 = object : Migration(2, 3) {
override fun migrate(db: SupportSQLiteDatabase) {
db.delete(CoreRepository.TABLE, "certificate IS NULL", null)
}
}

View File

@ -31,7 +31,7 @@ internal data class CoreRepository(
val formatVersion: IndexFormatVersion?,
val maxAge: Int?,
val description: LocalizedTextV2 = emptyMap(),
val certificate: String?,
val certificate: String,
) {
internal companion object {
const val TABLE = "CoreRepository"
@ -47,7 +47,7 @@ internal fun RepoV2.toCoreRepository(
repoId: Long = 0,
version: Long,
formatVersion: IndexFormatVersion? = null,
certificate: String? = null,
certificate: String,
) = CoreRepository(
repoId = repoId,
name = name,
@ -99,7 +99,7 @@ public data class Repository internal constructor(
address: String,
timestamp: Long,
formatVersion: IndexFormatVersion,
certificate: String?,
certificate: String,
version: Long,
weight: Int,
lastUpdated: Long,
@ -135,7 +135,7 @@ public data class Repository internal constructor(
public val timestamp: Long get() = repository.timestamp
public val version: Long get() = repository.version ?: 0
public val formatVersion: IndexFormatVersion? get() = repository.formatVersion
public val certificate: String? get() = repository.certificate
public val certificate: String get() = repository.certificate
/**
* True if this repository is an archive repo.

View File

@ -165,11 +165,13 @@ internal interface RepositoryDaoInt : RepositoryDao {
}
@Transaction
@VisibleForTesting
@Deprecated("Use insert instead")
fun insertEmptyRepo(
address: String,
username: String? = null,
password: String? = null,
certificate: String = "6789" // just used for testing
): Long {
val repo = CoreRepository(
name = mapOf("en-US" to address),
@ -179,7 +181,7 @@ internal interface RepositoryDaoInt : RepositoryDao {
version = null,
formatVersion = null,
maxAge = null,
certificate = null,
certificate = certificate,
)
val repoId = insertOrReplace(repo)
val currentMinWeight = getMinRepositoryWeight()
@ -197,7 +199,12 @@ internal interface RepositoryDaoInt : RepositoryDao {
@Transaction
@VisibleForTesting
fun insertOrReplace(repository: RepoV2, version: Long = 0): Long {
val repoId = insertOrReplace(repository.toCoreRepository(version = version))
val repoId = insertOrReplace(
repository.toCoreRepository(
version = version,
certificate = "0123", // just for testing
)
)
val currentMinWeight = getMinRepositoryWeight()
val repositoryPreferences = RepositoryPreferences(repoId, currentMinWeight - 2)
insert(repositoryPreferences)
@ -258,9 +265,9 @@ internal interface RepositoryDaoInt : RepositoryDao {
repository: RepoV2,
version: Long,
formatVersion: IndexFormatVersion,
certificate: String?,
) {
update(repository.toCoreRepository(repoId, version, formatVersion, certificate))
val repo = getRepository(repoId) ?: error("Repo with id $repoId did not exist")
update(repository.toCoreRepository(repoId, version, formatVersion, repo.certificate))
insertRepoTables(repoId, repository)
}
@ -274,16 +281,6 @@ internal interface RepositoryDaoInt : RepositoryDao {
@Update
fun updateRepository(repo: CoreRepository): Int
/**
* Updates the certificate for the [Repository] with the given [repoId].
* This should be used for V1 index updating where we only get the full cert
* after reading the entire index file.
* V2 index should use [update] instead as there the certificate is known
* before reading full index.
*/
@Query("UPDATE ${CoreRepository.TABLE} SET certificate = :certificate WHERE repoId = :repoId")
fun updateRepository(repoId: Long, certificate: String)
@Update
fun updateRepositoryPreferences(preferences: RepositoryPreferences)

View File

@ -56,24 +56,11 @@ public abstract class IndexUpdater {
*/
public abstract val formatVersion: IndexFormatVersion
/**
* Updates a new [repo] for the first time.
*/
public fun updateNewRepo(
repo: Repository,
expectedSigningFingerprint: String?,
): IndexUpdateResult = catchExceptions {
update(repo, null, expectedSigningFingerprint)
}
/**
* Updates an existing [repo] with a known [Repository.certificate].
*/
public fun update(
repo: Repository,
): IndexUpdateResult = catchExceptions {
require(repo.certificate != null) { "Repo ${repo.address} had no certificate" }
update(repo, repo.certificate, null)
public fun update(repo: Repository): IndexUpdateResult = catchExceptions {
updateRepo(repo)
}
private fun catchExceptions(block: () -> IndexUpdateResult): IndexUpdateResult {
@ -86,11 +73,7 @@ public abstract class IndexUpdater {
}
}
protected abstract fun update(
repo: Repository,
certificate: String?,
fingerprint: String?,
): IndexUpdateResult
protected abstract fun updateRepo(repo: Repository): IndexUpdateResult
}
internal fun Downloader.setIndexUpdateListener(

View File

@ -51,29 +51,8 @@ public class RepoUpdater(
/**
* Updates the given [repo].
* If [Repository.certificate] is null,
* the repo is considered to be new this being the first update.
*/
public fun update(
repo: Repository,
fingerprint: String? = null,
): IndexUpdateResult {
return if (repo.certificate == null) {
// This is a new repo without a certificate
updateNewRepo(repo, fingerprint)
} else {
update(repo)
}
}
private fun updateNewRepo(
repo: Repository,
expectedSigningFingerprint: String?,
): IndexUpdateResult = update(repo) { updater ->
updater.updateNewRepo(repo, expectedSigningFingerprint)
}
private fun update(repo: Repository): IndexUpdateResult = update(repo) { updater ->
public fun update(repo: Repository): IndexUpdateResult = update(repo) { updater ->
updater.update(repo)
}

View File

@ -35,11 +35,7 @@ public class IndexV1Updater(
public override val formatVersion: IndexFormatVersion = ONE
private val db: FDroidDatabaseInt = database as FDroidDatabaseInt
override fun update(
repo: Repository,
certificate: String?,
fingerprint: String?,
): IndexUpdateResult {
override fun updateRepo(repo: Repository): IndexUpdateResult {
// Normally, we shouldn't allow repository downgrades and assert the condition below.
// However, F-Droid is concerned that late v2 bugs will require users to downgrade to v1,
// as it happened already with the migration from v0 to v1.
@ -61,21 +57,16 @@ public class IndexV1Updater(
if (!downloader.hasChanged()) return IndexUpdateResult.Unchanged
val eTag = downloader.cacheTag
val verifier = IndexV1Verifier(file, certificate, fingerprint)
val verifier = IndexV1Verifier(file, repo.certificate, null)
db.runInTransaction {
val (cert, _) = verifier.getStreamAndVerify { inputStream ->
verifier.getStreamAndVerify { inputStream ->
listener?.onUpdateProgress(repo, 0, 0)
val streamReceiver = DbV1StreamReceiver(db, repo.repoId, compatibilityChecker)
val streamProcessor =
IndexV1StreamProcessor(streamReceiver, certificate, repo.timestamp)
val streamProcessor = IndexV1StreamProcessor(streamReceiver, repo.timestamp)
streamProcessor.process(inputStream)
}
// update certificate, if we didn't have any before
val repoDao = db.getRepositoryDao()
if (certificate == null) {
repoDao.updateRepository(repo.repoId, cert)
}
// update RepositoryPreferences with timestamp and ETag (for v1)
val repoDao = db.getRepositoryDao()
val updatedPrefs = repo.preferences.copy(
lastUpdated = System.currentTimeMillis(),
lastETag = eTag,

View File

@ -34,12 +34,8 @@ public class IndexV2Updater(
public override val formatVersion: IndexFormatVersion = TWO
private val db: FDroidDatabaseInt = database as FDroidDatabaseInt
override fun update(
repo: Repository,
certificate: String?,
fingerprint: String?,
): IndexUpdateResult {
val (cert, entry) = getCertAndEntry(repo, certificate, fingerprint)
override fun updateRepo(repo: Repository): IndexUpdateResult {
val (_, entry) = getCertAndEntry(repo, repo.certificate)
// don't process repos that we already did process in the past
if (entry.timestamp <= repo.timestamp) return IndexUpdateResult.Unchanged
// get diff, if available
@ -47,7 +43,7 @@ public class IndexV2Updater(
return if (diff == null || repo.formatVersion == ONE) {
// no diff found (or this is upgrade from v1 repo), so do full index update
val streamReceiver = DbV2StreamReceiver(db, repo.repoId, compatibilityChecker)
val streamProcessor = IndexV2FullStreamProcessor(streamReceiver, cert)
val streamProcessor = IndexV2FullStreamProcessor(streamReceiver)
processStream(repo, entry.index, entry.version, streamProcessor)
} else {
// use available diff
@ -57,11 +53,7 @@ public class IndexV2Updater(
}
}
private fun getCertAndEntry(
repo: Repository,
certificate: String?,
fingerprint: String?,
): Pair<String, Entry> {
private fun getCertAndEntry(repo: Repository, certificate: String): Pair<String, Entry> {
val file = tempFileProvider.createTempFile()
val downloader = downloaderFactory.createWithTryFirstMirror(
repo = repo,
@ -73,7 +65,7 @@ public class IndexV2Updater(
}
try {
downloader.download()
val verifier = EntryVerifier(file, certificate, fingerprint)
val verifier = EntryVerifier(file, certificate, null)
return verifier.getStreamAndVerify { inputStream ->
IndexParser.parseEntry(inputStream)
}

View File

@ -366,7 +366,7 @@ internal class RepoAdder(
address = uri.toString(),
timestamp = -1L,
formatVersion = indexFormatVersion,
certificate = null,
certificate = "This is fake and will be replaced by real cert before saving in DB.",
version = 0L,
weight = 0,
lastUpdated = -1L,

View File

@ -56,8 +56,8 @@ internal class RepoV2Fetcher(
log.info { "Downloaded entry, now streaming index..." }
val streamReceiver = RepoV2StreamReceiver(receiver, repo.username, repo.password)
val streamProcessor = IndexV2FullStreamProcessor(streamReceiver, cert)
val streamReceiver = RepoV2StreamReceiver(receiver, cert, repo.username, repo.password)
val streamProcessor = IndexV2FullStreamProcessor(streamReceiver)
val digestInputStream = if (uri.scheme?.startsWith("http") == true) {
// stream index for http(s) downloads
val indexRequest = DownloadRequest(

View File

@ -19,6 +19,7 @@ import org.fdroid.index.v2.RepoV2
internal open class RepoV2StreamReceiver(
private val receiver: RepoPreviewReceiver,
private val certificate: String,
private val username: String?,
private val password: String?,
) : IndexV2StreamReceiver {
@ -28,7 +29,7 @@ internal open class RepoV2StreamReceiver(
repo: RepoV2,
version: Long,
formatVersion: IndexFormatVersion,
certificate: String?,
certificate: String,
username: String?,
password: String?,
) = Repository(
@ -80,7 +81,7 @@ internal open class RepoV2StreamReceiver(
private val locales: LocaleListCompat = getLocales(Resources.getSystem().configuration)
override fun receive(repo: RepoV2, version: Long, certificate: String) {
override fun receive(repo: RepoV2, version: Long) {
receiver.onRepoReceived(
getRepository(
repo = repo,

View File

@ -27,25 +27,25 @@ internal class DbV2StreamReceiverTest {
timestamp = 42L,
)
every { db.getRepositoryDao() } returns mockk(relaxed = true)
dbV2StreamReceiver.receive(repoV2, 42L, "cert")
dbV2StreamReceiver.receive(repoV2, 42L)
// icon file without leading / does not pass
val repoV2NoSlash =
repoV2.copy(icon = mapOf("en" to FileV2(name = "foo", sha256 = "bar", size = 23L)))
assertFailsWith<SerializationException> {
dbV2StreamReceiver.receive(repoV2NoSlash, 42L, "cert")
dbV2StreamReceiver.receive(repoV2NoSlash, 42L)
}
// icon file without sha256 hash fails
val repoNoSha256 = repoV2.copy(icon = mapOf("en" to FileV2(name = "/foo", size = 23L)))
assertFailsWith<SerializationException> {
dbV2StreamReceiver.receive(repoNoSha256, 42L, "cert")
dbV2StreamReceiver.receive(repoNoSha256, 42L)
}
// icon file without size fails
val repoNoSize = repoV2.copy(icon = mapOf("en" to FileV2(name = "/foo", sha256 = "bar")))
assertFailsWith<SerializationException> {
dbV2StreamReceiver.receive(repoNoSize, 42L, "cert")
dbV2StreamReceiver.receive(repoNoSize, 42L)
}
}
}

View File

@ -38,7 +38,6 @@ import java.io.InputStream
@OptIn(ExperimentalSerializationApi::class)
public class IndexV1StreamProcessor(
private val indexStreamReceiver: IndexV1StreamReceiver,
private val certificate: String?,
private val lastTimestamp: Long,
private val locale: String = DEFAULT_LOCALE,
private val json: Json = IndexParser.json,
@ -98,7 +97,7 @@ public class IndexV1StreamProcessor(
categories = emptyMap(),
releaseChannels = emptyMap()
)
indexStreamReceiver.receive(repoV2, repo.version.toLong(), certificate)
indexStreamReceiver.receive(repoV2, repo.version.toLong())
}
private fun deserializeRequests(decoder: JsonDecoder, index: Int) {

View File

@ -16,7 +16,6 @@ import java.io.InputStream
@OptIn(ExperimentalSerializationApi::class)
public class IndexV2FullStreamProcessor(
private val indexStreamReceiver: IndexV2StreamReceiver,
private val certificate: String,
private val json: Json = IndexParser.json,
) : IndexV2StreamProcessor {
@ -68,7 +67,7 @@ public class IndexV2FullStreamProcessor(
private fun deserializeRepo(decoder: JsonDecoder, index: Int) {
require(index == descriptor.getElementIndex("repo"))
val repo = decoder.decodeSerializableValue(RepoV2.serializer())
indexStreamReceiver.receive(repo, version, certificate)
indexStreamReceiver.receive(repo, version)
}
private fun deserializePackages(decoder: JsonDecoder, index: Int) {

View File

@ -125,7 +125,7 @@ internal class IndexV1StreamProcessorTest {
) {
val file = File(filePath)
val testStreamReceiver = TestStreamReceiver()
val streamProcessor = IndexV1StreamProcessor(testStreamReceiver, null, lastTimestamp)
val streamProcessor = IndexV1StreamProcessor(testStreamReceiver, lastTimestamp)
FileInputStream(file).use { streamProcessor.process(it) }
assertEquals(indexV2.repo, testStreamReceiver.repo)
assertEquals(indexV2.packages, testStreamReceiver.packages)
@ -133,7 +133,7 @@ internal class IndexV1StreamProcessorTest {
private fun testStreamError(index: String) {
val testStreamReceiver = TestStreamReceiver()
val streamProcessor = IndexV1StreamProcessor(testStreamReceiver, null, -1)
val streamProcessor = IndexV1StreamProcessor(testStreamReceiver, -1)
ByteArrayInputStream(index.encodeToByteArray()).use { streamProcessor.process(it) }
assertNull(testStreamReceiver.repo)
assertEquals(0, testStreamReceiver.packages.size)
@ -144,7 +144,7 @@ internal class IndexV1StreamProcessorTest {
var repo: RepoV2? = null
val packages = HashMap<String, PackageV2>()
override fun receive(repo: RepoV2, version: Long, certificate: String?) {
override fun receive(repo: RepoV2, version: Long) {
this.repo = repo
}

View File

@ -7,7 +7,6 @@ import org.fdroid.test.TestDataEmptyV2
import org.fdroid.test.TestDataMaxV2
import org.fdroid.test.TestDataMidV2
import org.fdroid.test.TestDataMinV2
import org.fdroid.test.TestUtils.getRandomString
import org.junit.Rule
import org.junit.Test
import org.junit.rules.TemporaryFolder
@ -108,8 +107,7 @@ internal class IndexV2FullStreamProcessorTest {
private fun testStreamProcessing(filePath: String, index: IndexV2, expectedNumApps: Int) {
val file = File(filePath)
val testStreamReceiver = TestStreamReceiver()
val certificate = getRandomString()
val streamProcessor = IndexV2FullStreamProcessor(testStreamReceiver, certificate)
val streamProcessor = IndexV2FullStreamProcessor(testStreamReceiver)
var totalApps = 0
FileInputStream(file).use {
streamProcessor.process(42, it) { numAppsProcessed ->
@ -119,15 +117,13 @@ internal class IndexV2FullStreamProcessorTest {
assertTrue(testStreamReceiver.calledOnStreamEnded)
assertEquals(index.repo, testStreamReceiver.repo)
assertEquals(certificate, testStreamReceiver.certificate)
assertEquals(index.packages, testStreamReceiver.packages)
assertEquals(expectedNumApps, totalApps)
}
private fun testStreamError(str: String) {
val testStreamReceiver = TestStreamReceiver()
val certificate = getRandomString()
val streamProcessor = IndexV2FullStreamProcessor(testStreamReceiver, certificate)
val streamProcessor = IndexV2FullStreamProcessor(testStreamReceiver)
var totalApps = 0
ByteArrayInputStream(str.encodeToByteArray()).use {
streamProcessor.process(42, it) { numAppsProcessed ->
@ -136,20 +132,17 @@ internal class IndexV2FullStreamProcessorTest {
}
assertTrue(testStreamReceiver.calledOnStreamEnded)
assertEquals(certificate, testStreamReceiver.certificate)
assertEquals(0, testStreamReceiver.packages.size)
assertEquals(0, totalApps)
}
private open class TestStreamReceiver : IndexV2StreamReceiver {
var repo: RepoV2? = null
var certificate: String? = null
val packages = HashMap<String, PackageV2>()
var calledOnStreamEnded: Boolean = false
override fun receive(repo: RepoV2, version: Long, certificate: String) {
override fun receive(repo: RepoV2, version: Long) {
this.repo = repo
this.certificate = certificate
}
override fun receive(packageName: String, p: PackageV2) {

View File

@ -10,7 +10,7 @@ import org.fdroid.index.v2.RepoV2
@Deprecated("Use IndexV2 instead")
public interface IndexV1StreamReceiver {
public fun receive(repo: RepoV2, version: Long, certificate: String?)
public fun receive(repo: RepoV2, version: Long)
public fun receive(packageName: String, m: MetadataV2)
public fun receive(packageName: String, v: Map<String, PackageVersionV2>)

View File

@ -6,7 +6,7 @@ public interface IndexV2StreamReceiver {
* Receives the [RepoV2] from the index stream.
* Attention: This might get called after receiving packages.
*/
public fun receive(repo: RepoV2, version: Long, certificate: String)
public fun receive(repo: RepoV2, version: Long)
/**
* Receives one [PackageV2] from the index stream.