Detect office files for locking on new upload. Notify FolderWatcher.

Signed-off-by: alex-z <blackslayer4@gmail.com>
This commit is contained in:
alex-z 2024-04-03 18:10:49 +02:00
parent d6ed67806e
commit dbde9e3a2b
9 changed files with 329 additions and 102 deletions

View File

@ -1598,6 +1598,7 @@ void Folder::registerFolderWatcher()
connect(_folderWatcher.data(), &FolderWatcher::filesLockImposed, this, &Folder::slotFilesLockImposed, Qt::UniqueConnection);
_folderWatcher->init(path());
_folderWatcher->startNotificatonTest(path() + QLatin1String(".nextcloudsync.log"));
connect(_engine.data(), &SyncEngine::lockFileDetected, _folderWatcher.data(), &FolderWatcher::slotLockFileDetectedExternally);
}
void Folder::disconnectFolderWatcher()

View File

@ -41,34 +41,7 @@
namespace
{
const std::array<const char *, 2> lockFilePatterns = {{".~lock.", "~$"}};
constexpr auto lockChangeDebouncingTimerIntervalMs = 500;
QString filePathLockFilePatternMatch(const QString &path)
{
qCDebug(OCC::lcFolderWatcher) << "Checking if it is a lock file:" << path;
const auto pathSplit = path.split(QLatin1Char('/'), Qt::SkipEmptyParts);
if (pathSplit.isEmpty()) {
return {};
}
QString lockFilePatternFound;
for (const auto &lockFilePattern : lockFilePatterns) {
if (pathSplit.last().startsWith(lockFilePattern)) {
lockFilePatternFound = lockFilePattern;
break;
}
}
if (lockFilePatternFound.isEmpty()) {
return {};
}
qCDebug(OCC::lcFolderWatcher) << "Found a lock file with prefix:" << lockFilePatternFound << "in path:" << path;
return lockFilePatternFound;
}
}
namespace OCC {
@ -185,6 +158,22 @@ int FolderWatcher::testLinuxWatchCount() const
#endif
}
void FolderWatcher::slotLockFileDetectedExternally(const QString &lockFile)
{
qCInfo(lcFolderWatcher) << "Lock file detected externally, probably a newly-uploaded office file: " << lockFile;
changeDetected(lockFile);
}
void FolderWatcher::setShouldWatchForFileUnlocking(bool shouldWatchForFileUnlocking)
{
_shouldWatchForFileUnlocking = shouldWatchForFileUnlocking;
}
int FolderWatcher::lockChangeDebouncingTimout() const
{
return _lockChangeDebouncingTimer.interval();
}
void FolderWatcher::changeDetected(const QString &path)
{
QFileInfo fileInfo(path);
@ -220,17 +209,17 @@ void FolderWatcher::changeDetected(const QStringList &paths)
_testNotificationPath.clear();
}
const auto lockFileNamePattern = filePathLockFilePatternMatch(path);
const auto checkResult = lockFileTargetFilePath(path,lockFileNamePattern);
const auto lockFileNamePattern = FileSystem::filePathLockFilePatternMatch(path);
const auto checkResult = FileSystem::lockFileTargetFilePath(path, lockFileNamePattern);
if (_shouldWatchForFileUnlocking) {
// Lock file has been deleted, file now unlocked
if (checkResult.type == FileLockingInfo::Type::Unlocked && !checkResult.path.isEmpty()) {
if (checkResult.type == FileSystem::FileLockingInfo::Type::Unlocked && !checkResult.path.isEmpty()) {
_lockedFiles.remove(checkResult.path);
_unlockedFiles.insert(checkResult.path);
}
}
if (checkResult.type == FileLockingInfo::Type::Locked && !checkResult.path.isEmpty()) {
if (checkResult.type == FileSystem::FileLockingInfo::Type::Locked && !checkResult.path.isEmpty()) {
_unlockedFiles.remove(checkResult.path);
_lockedFiles.insert(checkResult.path);
}
@ -272,62 +261,4 @@ void FolderWatcher::folderAccountCapabilitiesChanged()
_shouldWatchForFileUnlocking = _folder->accountState()->account()->capabilities().filesLockAvailable();
}
FolderWatcher::FileLockingInfo FolderWatcher::lockFileTargetFilePath(const QString &path, const QString &lockFileNamePattern) const
{
FileLockingInfo result;
if (lockFileNamePattern.isEmpty()) {
return result;
}
const auto lockFilePathWithoutPrefix = QString(path).replace(lockFileNamePattern, QStringLiteral(""));
auto lockFilePathWithoutPrefixSplit = lockFilePathWithoutPrefix.split(QLatin1Char('.'));
if (lockFilePathWithoutPrefixSplit.size() < 2) {
return result;
}
auto extensionSanitized = lockFilePathWithoutPrefixSplit.takeLast().toStdString();
// remove possible non-alphabetical characters at the end of the extension
extensionSanitized.erase(
std::remove_if(extensionSanitized.begin(), extensionSanitized.end(), [](const auto &ch) {
return !std::isalnum(ch);
}),
extensionSanitized.end()
);
lockFilePathWithoutPrefixSplit.push_back(QString::fromStdString(extensionSanitized));
const auto lockFilePathWithoutPrefixNew = lockFilePathWithoutPrefixSplit.join(QLatin1Char('.'));
qCDebug(lcFolderWatcher) << "Assumed locked/unlocked file path" << lockFilePathWithoutPrefixNew << "Going to try to find matching file";
auto splitFilePath = lockFilePathWithoutPrefixNew.split(QLatin1Char('/'));
if (splitFilePath.size() > 1) {
const auto lockFileNameWithoutPrefix = splitFilePath.takeLast();
// some software will modify lock file name such that it does not correspond to original file (removing some symbols from the name, so we will search
// for a matching file
result.path = findMatchingUnlockedFileInDir(splitFilePath.join(QLatin1Char('/')), lockFileNameWithoutPrefix);
}
if (result.path.isEmpty() || !QFile::exists(result.path)) {
result.path.clear();
return result;
}
result.type = QFile::exists(path) ? FileLockingInfo::Type::Locked : FileLockingInfo::Type::Unlocked;
return result;
}
QString FolderWatcher::findMatchingUnlockedFileInDir(const QString &dirPath, const QString &lockFileName) const
{
QString foundFilePath;
const QDir dir(dirPath);
const auto entryList = dir.entryInfoList(QDir::Files);
for (const auto &candidateUnlockedFileInfo : entryList) {
if (candidateUnlockedFileInfo.fileName().contains(lockFileName)) {
foundFilePath = candidateUnlockedFileInfo.absoluteFilePath();
break;
}
}
return foundFilePath;
}
} // namespace OCC

View File

@ -50,12 +50,6 @@ class FolderWatcher : public QObject
{
Q_OBJECT
struct FileLockingInfo {
enum class Type { Unset = -1, Locked, Unlocked };
QString path;
Type type = Type::Unset;
};
public:
// Construct, connect signals, call init()
explicit FolderWatcher(Folder *folder = nullptr);
@ -86,6 +80,11 @@ public:
/// For testing linux behavior only
[[nodiscard]] int testLinuxWatchCount() const;
void slotLockFileDetectedExternally(const QString &lockFile);
void setShouldWatchForFileUnlocking(bool shouldWatchForFileUnlocking);
[[nodiscard]] int lockChangeDebouncingTimout() const;
signals:
/** Emitted when one of the watched directories or one
* of the contained files is changed. */
@ -101,8 +100,6 @@ signals:
*/
void filesLockImposed(const QSet<QString> &files);
void lockFilesFound(const QSet<QString> &files);
void lockedFilesFound(const QSet<QString> &files);
/**
@ -145,11 +142,6 @@ private:
void appendSubPaths(QDir dir, QStringList& subPaths);
[[nodiscard]] FileLockingInfo lockFileTargetFilePath(const QString &path, const QString &lockFileNamePattern) const;
[[nodiscard]] QString findMatchingUnlockedFileInDir(const QString &dirPath, const QString &lockFileName) const;
QString findMatchingUnlockedFileInDir(const QString &dirPath, const QString &lockFileName);
/* Check if the path should be ignored by the FolderWatcher. */
[[nodiscard]] bool pathIsIgnored(const QString &path) const;

View File

@ -25,12 +25,132 @@
#include <QDirIterator>
#include <QCoreApplication>
#include <array>
#include <string_view>
#ifdef Q_OS_WIN
#include <securitybaseapi.h>
#include <sddl.h>
#endif
namespace
{
constexpr std::array<const char *, 2> lockFilePatterns = {{".~lock.", "~$"}};
constexpr std::array<std::string_view, 8> officeFileExtensions = {"doc", "docx", "xls", "xlsx", "ppt", "pptx", "odt", "odp"};
// iterates through the dirPath to find the matching fileName
QString findMatchingUnlockedFileInDir(const QString &dirPath, const QString &lockFileName)
{
QString foundFilePath;
const QDir dir(dirPath);
const auto entryList = dir.entryInfoList(QDir::Files);
for (const auto &candidateUnlockedFileInfo : entryList) {
const auto candidateUnlockFileName = candidateUnlockedFileInfo.fileName();
const auto lockFilePatternFoundIt = std::find_if(std::cbegin(lockFilePatterns), std::cend(lockFilePatterns), [&candidateUnlockFileName](std::string_view pattern) {
return candidateUnlockFileName.contains(QString::fromStdString(std::string(pattern)));
});
if (lockFilePatternFoundIt != std::cend(lockFilePatterns)) {
continue;
}
if (candidateUnlockFileName.contains(lockFileName)) {
foundFilePath = candidateUnlockedFileInfo.absoluteFilePath();
break;
}
}
return foundFilePath;
}
}
namespace OCC {
QString FileSystem::filePathLockFilePatternMatch(const QString &path)
{
qCDebug(OCC::lcFileSystem) << "Checking if it is a lock file:" << path;
const auto pathSplit = path.split(QLatin1Char('/'), Qt::SkipEmptyParts);
if (pathSplit.isEmpty()) {
return {};
}
QString lockFilePatternFound;
for (const auto &lockFilePattern : lockFilePatterns) {
if (pathSplit.last().startsWith(lockFilePattern)) {
lockFilePatternFound = lockFilePattern;
break;
}
}
if (!lockFilePatternFound.isEmpty()) {
qCDebug(OCC::lcFileSystem) << "Found a lock file with prefix:" << lockFilePatternFound << "in path:" << path;
}
return lockFilePatternFound;
}
bool FileSystem::isMatchingOfficeFileExtension(const QString &path)
{
const auto pathSplit = path.split(QLatin1Char('.'));
const auto extension = pathSplit.size() > 1 ? pathSplit.last().toStdString() : std::string{};
return std::find(std::cbegin(officeFileExtensions), std::cend(officeFileExtensions), extension) != std::cend(officeFileExtensions);
}
FileSystem::FileLockingInfo FileSystem::lockFileTargetFilePath(const QString &lockFilePath, const QString &lockFileNamePattern)
{
FileLockingInfo result;
if (lockFileNamePattern.isEmpty()) {
return result;
}
const auto lockFilePathWithoutPrefix = QString(lockFilePath).replace(lockFileNamePattern, QStringLiteral(""));
auto lockFilePathWithoutPrefixSplit = lockFilePathWithoutPrefix.split(QLatin1Char('.'));
if (lockFilePathWithoutPrefixSplit.size() < 2) {
return result;
}
auto extensionSanitized = lockFilePathWithoutPrefixSplit.takeLast().toStdString();
// remove possible non-alphabetical characters at the end of the extension
extensionSanitized.erase(std::remove_if(extensionSanitized.begin(),
extensionSanitized.end(),
[](const auto &ch) {
return !std::isalnum(ch);
}),
extensionSanitized.end());
lockFilePathWithoutPrefixSplit.push_back(QString::fromStdString(extensionSanitized));
const auto lockFilePathWithoutPrefixNew = lockFilePathWithoutPrefixSplit.join(QLatin1Char('.'));
qCDebug(lcFileSystem) << "Assumed locked/unlocked file path" << lockFilePathWithoutPrefixNew << "Going to try to find matching file";
auto splitFilePath = lockFilePathWithoutPrefixNew.split(QLatin1Char('/'));
if (splitFilePath.size() > 1) {
const auto lockFileNameWithoutPrefix = splitFilePath.takeLast();
// some software will modify lock file name such that it does not correspond to original file (removing some symbols from the name, so we will
// search for a matching file
result.path = findMatchingUnlockedFileInDir(splitFilePath.join(QLatin1Char('/')), lockFileNameWithoutPrefix);
}
if (result.path.isEmpty() || !QFile::exists(result.path)) {
result.path.clear();
return result;
}
result.type = QFile::exists(lockFilePath) ? FileLockingInfo::Type::Locked : FileLockingInfo::Type::Unlocked;
return result;
}
QStringList FileSystem::findAllLockFilesInDir(const QString &dirPath)
{
QStringList results;
const QDir dir(dirPath);
const auto entryList = dir.entryInfoList(QDir::Files | QDir::Hidden | QDir::NoDotAndDotDot);
for (const auto &candidateLockFile : entryList) {
const auto filePath = candidateLockFile.filePath();
const auto isLockFile = !filePathLockFilePatternMatch(filePath).isEmpty();
if (isLockFile) {
results.push_back(filePath);
}
}
return results;
}
bool FileSystem::fileEquals(const QString &fn1, const QString &fn2)
{

View File

@ -20,6 +20,7 @@
#include "common/filesystembase.h"
#include <QString>
#include <QStringList>
#include <ctime>
#include <functional>
@ -42,6 +43,20 @@ class SyncJournal;
* @brief This file contains file system helper
*/
namespace FileSystem {
struct OWNCLOUDSYNC_EXPORT FileLockingInfo {
enum class Type { Unset = -1, Locked, Unlocked };
QString path;
Type type = Type::Unset;
};
// match file path with lock pattern
QString OWNCLOUDSYNC_EXPORT filePathLockFilePatternMatch(const QString &path);
// check if it is an office file (by extension), ONLY call it for files
bool OWNCLOUDSYNC_EXPORT isMatchingOfficeFileExtension(const QString &path);
// finds and fetches FileLockingInfo for the corresponding file that we are locking/unlocking
FileLockingInfo OWNCLOUDSYNC_EXPORT lockFileTargetFilePath(const QString &lockFilePath, const QString &lockFileNamePattern);
// lists all files matching a lockfile pattern in dirPath
QStringList OWNCLOUDSYNC_EXPORT findAllLockFilesInDir(const QString &dirPath);
/**
* @brief compare two files with given filename and return true if they have the same content

View File

@ -932,6 +932,33 @@ void SyncEngine::slotCleanPollsJobAborted(const QString &error, const ErrorCateg
finalize(false);
}
void SyncEngine::detectFileLock(const SyncFileItemPtr &item)
{
const auto isNewlyUploadedFile = !item->isDirectory() &&
item->_instruction == CSYNC_INSTRUCTION_NEW &&
item->_direction == SyncFileItem::Up && item->_status == SyncFileItem::Success;
if (isNewlyUploadedFile && item->_locked != SyncFileItem::LockStatus::LockedItem && _account->capabilities().filesLockAvailable() &&
FileSystem::isMatchingOfficeFileExtension(item->_file)) {
{
SyncJournalFileRecord rec;
if (!_journal->getFileRecord(item->_file, &rec) || !rec.isValid()) {
qCWarning(lcEngine) << "Newly-created office file just uploaded but not in sync journal. Not going to lock it." << item->_file;
return;
}
}
const auto localFilePath = _propagator->fullLocalPath(item->_file);
const auto allMatchingLockFiles = FileSystem::findAllLockFilesInDir(QFileInfo(localFilePath).absolutePath());
for (const auto &lockFilePath : allMatchingLockFiles) {
const auto checkResult = FileSystem::lockFileTargetFilePath(lockFilePath, FileSystem::filePathLockFilePatternMatch(lockFilePath));
if (checkResult.type == FileSystem::FileLockingInfo::Type::Locked && checkResult.path == localFilePath) {
qCInfo(lcEngine) << "Newly-created office file lock detected. Let FolderWatcher take it from here..." << item->_file;
emit lockFileDetected(lockFilePath);
}
}
}
}
void SyncEngine::setNetworkLimits(int upload, int download)
{
_uploadLimit = upload;
@ -954,6 +981,8 @@ void SyncEngine::slotItemCompleted(const SyncFileItemPtr &item, const ErrorCateg
emit transmissionProgress(*_progressInfo);
emit itemCompleted(item, category);
detectFileLock(item);
}
void SyncEngine::slotPropagationFinished(OCC::SyncFileItem::Status status)

View File

@ -195,6 +195,8 @@ signals:
*/
void seenLockedFile(const QString &fileName);
void lockFileDetected(const QString &lockFile);
private slots:
void slotFolderDiscovered(bool local, const QString &folder);
void slotRootEtagReceived(const QByteArray &, const QDateTime &time);
@ -215,6 +217,7 @@ private slots:
void slotPropagationFinished(SyncFileItem::Status status);
void slotProgress(const OCC::SyncFileItem &item, qint64 current);
void slotCleanPollsJobAborted(const QString &error, const OCC::ErrorCategory category);
void detectFileLock(const OCC::SyncFileItemPtr &item);
/** Records that a file was touched by a job. */
void slotAddTouchedFile(const QString &fn);

View File

@ -262,6 +262,121 @@ private slots:
mkdir(dir);
QVERIFY(waitForPathChanged(dir));
}
void testDetectLockFiles()
{
QStringList listOfOfficeFiles = {QString(_rootPath + "/document.docx"), QString(_rootPath + "/document.odt")};
std::sort(std::begin(listOfOfficeFiles), std::end(listOfOfficeFiles));
const QStringList listOfOfficeLockFiles = {QString(_rootPath + "/.~lock.document.docx#"), QString(_rootPath + "/.~lock.document.odt#")};
_watcher->setShouldWatchForFileUnlocking(true);
// verify that office files for locking got detected by the watcher
QScopedPointer<QSignalSpy> locksImposedSpy(new QSignalSpy(_watcher.data(), &FolderWatcher::filesLockImposed));
for (const auto &officeFile : listOfOfficeFiles) {
touch(officeFile);
QVERIFY(waitForPathChanged(officeFile));
}
for (const auto &officeLockFile : listOfOfficeLockFiles) {
touch(officeLockFile);
QVERIFY(waitForPathChanged(officeLockFile));
}
locksImposedSpy->wait(_watcher->lockChangeDebouncingTimout() + 100);
QCOMPARE(locksImposedSpy->count(), 1);
auto lockedOfficeFilesByWatcher = locksImposedSpy->takeFirst().at(0).value<QSet<QString>>().values();
std::sort(std::begin(lockedOfficeFilesByWatcher), std::end(lockedOfficeFilesByWatcher));
QCOMPARE(listOfOfficeFiles.size(), lockedOfficeFilesByWatcher.size());
for (int i = 0; i < listOfOfficeFiles.size(); ++i) {
QVERIFY(listOfOfficeFiles.at(i) == lockedOfficeFilesByWatcher.at(i));
}
// verify that office files for unlocking got detected by the watcher
QScopedPointer<QSignalSpy> locksReleasedSpy(new QSignalSpy(_watcher.data(), &FolderWatcher::filesLockReleased));
for (const auto &officeLockFile : listOfOfficeLockFiles) {
rm(officeLockFile);
QVERIFY(waitForPathChanged(officeLockFile));
}
locksReleasedSpy->wait(_watcher->lockChangeDebouncingTimout() + 100);
QCOMPARE(locksReleasedSpy->count(), 1);
auto unLockedOfficeFilesByWatcher = locksReleasedSpy->takeFirst().at(0).value<QSet<QString>>().values();
std::sort(std::begin(unLockedOfficeFilesByWatcher), std::end(unLockedOfficeFilesByWatcher));
QCOMPARE(listOfOfficeFiles.size(), unLockedOfficeFilesByWatcher.size());
for (int i = 0; i < listOfOfficeFiles.size(); ++i) {
QVERIFY(listOfOfficeFiles.at(i) == unLockedOfficeFilesByWatcher.at(i));
}
// cleanup
for (const auto &officeLockFile : listOfOfficeLockFiles) {
rm(officeLockFile);
}
for (const auto &officeFile : listOfOfficeFiles) {
rm(officeFile);
}
}
void testDetectLockFilesExternally()
{
QStringList listOfOfficeFiles = {QString(_rootPath + "/document.docx"), QString(_rootPath + "/document.odt")};
std::sort(std::begin(listOfOfficeFiles), std::end(listOfOfficeFiles));
const QStringList listOfOfficeLockFiles = {QString(_rootPath + "/.~lock.document.docx#"), QString(_rootPath + "/.~lock.document.odt#")};
for (const auto &officeFile : listOfOfficeFiles) {
touch(officeFile);
}
for (const auto &officeLockFile : listOfOfficeLockFiles) {
touch(officeLockFile);
}
_watcher.reset(new FolderWatcher);
_watcher->init(_rootPath);
_watcher->setShouldWatchForFileUnlocking(true);
_pathChangedSpy.reset(new QSignalSpy(_watcher.data(), &FolderWatcher::pathChanged));
QScopedPointer<QSignalSpy> locksImposedSpy(new QSignalSpy(_watcher.data(), &FolderWatcher::filesLockImposed));
QScopedPointer<QSignalSpy> locksReleasedSpy(new QSignalSpy(_watcher.data(), &FolderWatcher::filesLockReleased));
for (const auto &officeLockFile : listOfOfficeLockFiles) {
_watcher->slotLockFileDetectedExternally(officeLockFile);
}
// locked files detected
locksImposedSpy->wait(_watcher->lockChangeDebouncingTimout() + 100);
QCOMPARE(locksImposedSpy->count(), 1);
auto lockedOfficeFilesByWatcher = locksImposedSpy->takeFirst().at(0).value<QSet<QString>>().values();
std::sort(std::begin(lockedOfficeFilesByWatcher), std::end(lockedOfficeFilesByWatcher));
QCOMPARE(listOfOfficeFiles.size(), lockedOfficeFilesByWatcher.size());
for (int i = 0; i < listOfOfficeFiles.size(); ++i) {
QVERIFY(listOfOfficeFiles.at(i) == lockedOfficeFilesByWatcher.at(i));
}
// unlocked files detected
for (const auto &officeLockFile : listOfOfficeLockFiles) {
rm(officeLockFile);
}
locksReleasedSpy->wait(_watcher->lockChangeDebouncingTimout() + 100);
QCOMPARE(locksReleasedSpy->count(), 1);
auto unLockedOfficeFilesByWatcher = locksReleasedSpy->takeFirst().at(0).value<QSet<QString>>().values();
std::sort(std::begin(unLockedOfficeFilesByWatcher), std::end(unLockedOfficeFilesByWatcher));
QCOMPARE(listOfOfficeFiles.size(), unLockedOfficeFilesByWatcher.size());
for (int i = 0; i < listOfOfficeFiles.size(); ++i) {
QVERIFY(listOfOfficeFiles.at(i) == unLockedOfficeFilesByWatcher.at(i));
}
// cleanup
for (const auto &officeFile : listOfOfficeFiles) {
rm(officeFile);
}
for (const auto &officeLockFile : listOfOfficeLockFiles) {
rm(officeLockFile);
}
}
};
#ifdef Q_OS_MAC

View File

@ -750,6 +750,27 @@ private slots:
const auto localFileLocked = QFileInfo{fakeFolder.localPath() + u"A/a1"};
QVERIFY(!localFileLocked.isWritable());
}
void testLockFile_lockFile_detect_newly_uploaded()
{
const auto testFileName = QStringLiteral("document.docx");
const auto testLockFileName = QStringLiteral(".~lock.document.docx#");
const auto testDocumentsDirName = "documents";
FakeFolder fakeFolder{FileInfo{}};
fakeFolder.localModifier().mkdir(testDocumentsDirName);
fakeFolder.syncEngine().account()->setCapabilities({{"files", QVariantMap{{"locking", QByteArray{"1.0"}}}}});
QSignalSpy lockFileDetectedNewlyUploadedSpy(&fakeFolder.syncEngine(), &OCC::SyncEngine::lockFileDetected);
fakeFolder.localModifier().insert(testDocumentsDirName + QString("/") + testLockFileName);
fakeFolder.localModifier().insert(testDocumentsDirName + QString("/") + testFileName);
QVERIFY(fakeFolder.syncOnce());
QCOMPARE(lockFileDetectedNewlyUploadedSpy.count(), 1);
}
};
QTEST_GUILESS_MAIN(TestLockFile)