opensourcemirror
/
nextcloud-server
mirror of https://github.com/nextcloud/server.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #43574 from nextcloud/enh/noid/lazyappconfig-migrate-metadata 

enh(metadata): migrate metadata to lazy appconfig
This commit is contained in:
Maxence Lange 2024-04-17 16:20:33 -01:00 committed by GitHub
parent 3aa9c53a87 4e1814acd1
commit b4004a2582
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 55 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
lib/private/Files/Cache/Cache.php
View File

@ -171,7 +171,7 @@ class Cache implements ICache {
} elseif (!$data) {
return $data;
} else {
$data['metadata'] = $metadataQuery?->extractMetadata($data)->asArray() ?? [];
$data['metadata'] = $metadataQuery->extractMetadata($data)->asArray();
return self::cacheEntryFromData($data, $this->mimetypeLoader);
}
}
@ -243,7 +243,7 @@ class Cache implements ICache {
$result->closeCursor();
return array_map(function (array $data) use ($metadataQuery) {
$data['metadata'] = $metadataQuery?->extractMetadata($data)->asArray() ?? [];
$data['metadata'] = $metadataQuery->extractMetadata($data)->asArray();
return self::cacheEntryFromData($data, $this->mimetypeLoader);
}, $files);
}

6
lib/private/Files/Cache/CacheQueryBuilder.php
View File

@ -138,11 +138,11 @@ class CacheQueryBuilder extends QueryBuilder {
/**
* join metadata to current query builder and returns an helper
*
* @return IMetadataQuery|null NULL if no metadata have never been generated
* @return IMetadataQuery
*/
public function selectMetadata(): ?IMetadataQuery {
public function selectMetadata(): IMetadataQuery {
$metadataQuery = $this->filesMetadataManager->getMetadataQuery($this, $this->alias, 'fileid');
$metadataQuery?->retrieveMetadata();
$metadataQuery->retrieveMetadata();
return $metadataQuery;
}
}

7
lib/private/Files/Cache/QuerySearchHelper.php
View File

@ -195,12 +195,7 @@ class QuerySearchHelper {
$files = $result->fetchAll();
$rawEntries = array_map(function (array $data) use ($metadataQuery) {
// migrate to null safe ...
if ($metadataQuery === null) {
$data['metadata'] = [];
} else {
$data['metadata'] = $metadataQuery->extractMetadata($data)->asArray();
}
$data['metadata'] = $metadataQuery->extractMetadata($data)->asArray();
return Cache::cacheEntryFromData($data, $this->mimetypeLoader);
}, $files);

44
lib/private/FilesMetadata/FilesMetadataManager.php
View File

@ -51,8 +51,7 @@ use OCP\FilesMetadata\IFilesMetadataManager;
use OCP\FilesMetadata\IMetadataQuery;
use OCP\FilesMetadata\Model\IFilesMetadata;
use OCP\FilesMetadata\Model\IMetadataValueWrapper;
use OCP\IConfig;
use OCP\IDBConnection;
use OCP\IAppConfig;
use Psr\Log\LoggerInterface;
/**
@ -69,7 +68,7 @@ class FilesMetadataManager implements IFilesMetadataManager {
public function __construct(
private IEventDispatcher $eventDispatcher,
private IJobList $jobList,
private IConfig $config,
private IAppConfig $appConfig,
private LoggerInterface $logger,
private MetadataRequestService $metadataRequestService,
private IndexRequestService $indexRequestService,
@ -206,7 +205,7 @@ class FilesMetadataManager implements IFilesMetadataManager {
// update metadata types list
$current = $this->getKnownMetadata();
$current->import($filesMetadata->jsonSerialize(true));
$this->config->setAppValue('core', self::CONFIG_KEY, json_encode($current));
$this->appConfig->setValueArray('core', self::CONFIG_KEY, $current->jsonSerialize(), lazy: true);
}
/**
@ -235,7 +234,7 @@ class FilesMetadataManager implements IFilesMetadataManager {
* @param string $fileIdField alias of the field that contains file ids
*
* @inheritDoc
* @return IMetadataQuery|null
* @return IMetadataQuery
* @see IMetadataQuery
* @since 28.0.0
*/
@ -243,12 +242,8 @@ class FilesMetadataManager implements IFilesMetadataManager {
IQueryBuilder $qb,
string $fileTableAlias,
string $fileIdField
): ?IMetadataQuery {
if (!$this->metadataInitiated()) {
return null;
}
return new MetadataQuery($qb, $this->getKnownMetadata(), $fileTableAlias, $fileIdField);
): IMetadataQuery {
return new MetadataQuery($qb, $this, $fileTableAlias, $fileIdField);
}
/**
@ -263,8 +258,7 @@ class FilesMetadataManager implements IFilesMetadataManager {
$this->all = new FilesMetadata();
try {
$data = json_decode($this->config->getAppValue('core', self::CONFIG_KEY, '[]'), true, 127, JSON_THROW_ON_ERROR);
$this->all->import($data);
$this->all->import($this->appConfig->getValueArray('core', self::CONFIG_KEY, lazy: true));
} catch (JsonException) {
$this->logger->warning('issue while reading stored list of metadata. Advised to run ./occ files:scan --all --generate-metadata');
}
@ -310,7 +304,7 @@ class FilesMetadataManager implements IFilesMetadataManager {
}
$current->import([$key => ['type' => $type, 'indexed' => $indexed, 'editPermission' => $editPermission]]);
$this->config->setAppValue('core', self::CONFIG_KEY, json_encode($current));
$this->appConfig->setValueArray('core', self::CONFIG_KEY, $current->jsonSerialize(), lazy: true);
$this->all = $current;
}
@ -323,26 +317,4 @@ class FilesMetadataManager implements IFilesMetadataManager {
$eventDispatcher->addServiceListener(NodeWrittenEvent::class, MetadataUpdate::class);
$eventDispatcher->addServiceListener(CacheEntryRemovedEvent::class, MetadataDelete::class);
}
/**
* Will confirm that tables were created and store an app value to cache the result.
* Can be removed in 29 as this is to avoid strange situation when Nextcloud files were
* replaced but the upgrade was not triggered yet.
*
* @return bool
*/
private function metadataInitiated(): bool {
if ($this->config->getAppValue('core', self::MIGRATION_DONE, '0') === '1') {
return true;
}
$dbConnection = \OCP\Server::get(IDBConnection::class);
if ($dbConnection->tableExists(MetadataRequestService::TABLE_METADATA)) {
$this->config->setAppValue('core', self::MIGRATION_DONE, '1');
return true;
}
return false;
}
}

30
lib/private/FilesMetadata/MetadataQuery.php
View File

@ -31,9 +31,11 @@ use OC\FilesMetadata\Service\MetadataRequestService;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\FilesMetadata\Exceptions\FilesMetadataNotFoundException;
use OCP\FilesMetadata\Exceptions\FilesMetadataTypeException;
use OCP\FilesMetadata\IFilesMetadataManager;
use OCP\FilesMetadata\IMetadataQuery;
use OCP\FilesMetadata\Model\IFilesMetadata;
use OCP\FilesMetadata\Model\IMetadataValueWrapper;
use Psr\Log\LoggerInterface;
/**
* @inheritDoc
@ -43,12 +45,23 @@ class MetadataQuery implements IMetadataQuery {
private array $knownJoinedIndex = [];
public function __construct(
private IQueryBuilder $queryBuilder,
private IFilesMetadata $knownMetadata,
private IFilesMetadata|IFilesMetadataManager $manager,
private string $fileTableAlias = 'fc',
private string $fileIdField = 'fileid',
private string $alias = 'meta',
private string $aliasIndexPrefix = 'meta_index'
) {
if ($manager instanceof IFilesMetadata) {
/**
* Since 29, because knownMetadata is stored in lazy appconfig, it seems smarter
* to not call getKnownMetadata() at the load of this class as it is only needed
* in {@see getMetadataValueField}.
*
* FIXME: remove support for IFilesMetadata
*/
$logger = \OCP\Server::get(LoggerInterface::class);
$logger->debug('It is deprecated to use IFilesMetadata as second parameter when calling MetadataQuery::__construct()');
}
}
/**
@ -158,7 +171,20 @@ class MetadataQuery implements IMetadataQuery {
* @since 28.0.0
*/
public function getMetadataValueField(string $metadataKey): string {
return match ($this->knownMetadata->getType($metadataKey)) {
if ($this->manager instanceof IFilesMetadataManager) {
/**
* Since 29, because knownMetadata is stored in lazy appconfig, it seems smarter
* to not call getKnownMetadata() at the load of this class as it is only needed
* in this method.
*
* FIXME: keep only this line and remove support for previous IFilesMetadata in constructor
*/
$knownMetadata = $this->manager->getKnownMetadata();
} else {
$knownMetadata = $this->manager;
}
return match ($knownMetadata->getType($metadataKey)) {
IMetadataValueWrapper::TYPE_STRING => $this->joinedTableAlias($metadataKey) . '.meta_value_string',
IMetadataValueWrapper::TYPE_INT, IMetadataValueWrapper::TYPE_BOOL => $this->joinedTableAlias($metadataKey) . '.meta_value_int',
default => throw new FilesMetadataTypeException('metadata is not set as indexed'),

16
lib/public/FilesMetadata/IFilesMetadataManager.php
View File

@ -122,7 +122,7 @@ interface IFilesMetadataManager {
* @param string $fileTableAlias alias of the table that contains data about files
* @param string $fileIdField alias of the field that contains file ids
*
* @return IMetadataQuery|null NULL if table are not set yet or never used
* @return IMetadataQuery
* @see IMetadataQuery
* @since 28.0.0
*/
@ -130,22 +130,27 @@ interface IFilesMetadataManager {
IQueryBuilder $qb,
string $fileTableAlias,
string $fileIdField
): ?IMetadataQuery;
): IMetadataQuery;
/**
* returns all type of metadata currently available.
* The list is stored in a IFilesMetadata with null values but correct type.
*
* Note: this method loads lazy appconfig values.
*
* @return IFilesMetadata
* @since 28.0.0
*/
public function getKnownMetadata(): IFilesMetadata;
/**
* initiate a metadata key with its type.
* Initiate a metadata key with its type.
*
* The call is mandatory before using the metadata property in a webdav request.
* It is not needed to only use this method when the app is enabled: the method can be
* called each time during the app loading as the metadata will only be initiated if not known
* The call should be part of a migration/repair step and not be called on app's boot
* process as it is using lazy-appconfig value
*
* Note: this method loads lazy appconfig values.
*
* @param string $key metadata key
* @param string $type metadata type
@ -164,6 +169,7 @@ interface IFilesMetadataManager {
* @see IMetadataValueWrapper::EDIT_REQ_WRITE_PERMISSION
* @see IMetadataValueWrapper::EDIT_REQ_READ_PERMISSION
* @since 28.0.0
* @since 29.0.0 uses lazy config value - do not use this method out of repair steps
*/
public function initMetadata(string $key, string $type, bool $indexed, int $editPermission): void;
}

2
lib/public/FilesMetadata/IMetadataQuery.php
View File

@ -83,6 +83,8 @@ interface IMetadataQuery {
/**
* returns the name of the field for metadata string value to be used in query expressions
*
* Note: this method loads lazy appconfig values.
*
* @param string $metadataKey metadata key
*
* @return string table field