Index: sources/storage/FileHandler.cpp =================================================================== diff -u -r12e7b6dda53cf8db7707c7fa55dcf6137e7d6997 -rcf51c19d82d667644d7f70eaa521b43f908c7068 --- sources/storage/FileHandler.cpp (.../FileHandler.cpp) (revision 12e7b6dda53cf8db7707c7fa55dcf6137e7d6997) +++ sources/storage/FileHandler.cpp (.../FileHandler.cpp) (revision cf51c19d82d667644d7f70eaa521b43f908c7068) @@ -1,30 +1,29 @@ /*! * - * Copyright (c) 2019-2020 Diality Inc. - All Rights Reserved. + * Copyright (c) 2020-2023 Diality Inc. - All Rights Reserved. * \copyright * THIS CODE MAY NOT BE COPIED OR REPRODUCED IN ANY FORM, IN PART OR IN * WHOLE, WITHOUT THE EXPLICIT PERMISSION OF THE COPYRIGHT OWNER. * - * \file FileHandler.cpp - * \author (last) Behrouz NematiPour - * \date (last) 25-Aug-2020 - * \author (original) Behrouz NematiPour - * \date (original) 24-Sep-2019 + * \file FileHandler.cpp + * \author (last) Behrouz NematiPour + * \date (last) 22-Jan-2023 + * \author (original) Behrouz NematiPour + * \date (original) 26-Aug-2020 * */ #include "FileHandler.h" //Qt -#include #include #include #include -#include -#include #include + // Project -#include "Logger.h" +// #include "Logger.h" // logger should not be used in this class. +#include "StorageGlobals.h" // namespace using namespace Storage; @@ -35,23 +34,23 @@ * \param vFileName - Source file name * \param vContent - The content which is going to be written in the file. * \param vAppend - if set to true the content will be appended at the end of the file. - * \return false if file can't be opened. + * \return false if file cannot be opened. */ void FileHandler::errOut(const QString &vMessage) { static uint count; static QString mCritical; - // coco begin validated : This has been manually test. Needs file system access to produce errors for hundred times. + // disabled coco begin validated : This has been manually test. Needs file system access to produce errors for hundred times. if (mCritical != vMessage || !(count % 1000)) { - // coco end + // disabled coco end count = 0; mCritical = vMessage; - QTextStream err(stderr); + QTextStream err(stderr); err << "FS" << " " << QDate::currentDate().toString("yyyy_MM_dd") << " " << QTime::currentTime().toString("HH:mm:ss" ) << " " << mCritical - << endl; + << Qt::endl; } ++count; } @@ -63,22 +62,22 @@ * \param vFileName - Source file name * \param vContent - content to be written into file. * \param vAppend - append (true) or overwrite (false) - * \return false if file can't be opened for write. + * \return false if file cannot be opened for write. */ bool FileHandler::write(const QString &vFileName, const QString &vContent, bool vAppend) { QFile file(vFileName); QFile::OpenMode openMode = vAppend ? QFile::Text | QFile::Append : QFile::Text | QFile::WriteOnly; - // coco begin validated : This has been manually test. Needs file system access to make file the way it can't be opened for writing. + // disabled coco begin validated : This has been manually test. Needs file system access to make file the way it cannot be opened for writing. if (! file.open(openMode)) { - QString msg = QString("Can't open file for write (%1).Possible corrupted file system").arg(vFileName); - // here can't use LOG_XXXX because if the folder can't be created then the log can't be written. + QString msg = QString("Cannot open file for write (%1). Possible corrupted file system.").arg(vFileName); + // here cannot use LOG_XXXX because if the folder cannot be created then the log cannot be written. errOut (msg); return false; } - // coco end + // disabled coco end QTextStream out(&file); out << vContent; out.flush(); @@ -90,45 +89,116 @@ * \details reads file vFilename content into vContent variable. * \param vFileName - Source file name * \param vContent - The content of the file which will be set when done. - * \return false if file can't be opened. + * \return false if file cannot be opened. */ -bool FileHandler::read(const QString &vFileName, QString &vContent) +bool FileHandler::read(const QString &vFileName, QString &vContent, bool vAppend) { QFile file(vFileName); if (! file.open(QFile::Text | QFile::ReadOnly)) { - QString msg = QString("Can't open file for read (%1).Possible corrupted file system").arg(vFileName); - // here can't use LOG_XXXX because if the folder can't be created then the log can't be written. + QString msg = QString("Cannot open file for write (%1). Possible corrupted file system").arg(vFileName); + // here cannot use LOG_XXXX because if the folder cannot be created then the log cannot be written. errOut (msg); return false; } QTextStream in(&file); - vContent = in.readAll(); + if ( vAppend ) { + vContent += in.readAll(); + } + else { + vContent = in.readAll(); + } return true; } /*! + * \brief FileHandler::read + * \details reads the provided filename's JSON content into the vContent variable. + * \param vFileName - Source file name + * \param vContent - The content of the file which will be written to when done. + * \return false if file cannot be read or parsed + */ +bool FileHandler::read(const QString &vFileName, QJsonObject &vContent, QJsonParseError *error) +{ + QFile file(vFileName); + if (! file.open(QFile::Text | QFile::ReadOnly)) { + errOut(QObject::tr("Cannot open file for read (%1). Possible corrupted file system.").arg(vFileName)); + return false; + } + QTextStream in(&file); + QString content = in.readAll(); + QJsonParseError jsonParseError; + QJsonDocument doc = QJsonDocument::fromJson(content.toUtf8(), &jsonParseError); + if (jsonParseError.error) { + if (error) *error = jsonParseError; + return false; + } + vContent = doc.object(); + return true; +} + +/*! * \brief FileHandler::copyFolder * \details Copies all the file and folders recursively. * \param vSource - The source folder * \param vDestination - The destination folder - * \return Tue on successful execution. + * \return True on successful execution. * \note This method uses the Linux "cp -r vSource vDestination" command * Not a Thread-Safe. - * */ -int FileHandler::copyFolder(const QString &vSource, const QString &vDestination ) +int FileHandler::copyFolder(const QString &vSource, const QString &vDestination) { - // coco begin validated: This needs user interaction to export to USB device + // disabled coco begin validated: This needs user interaction to export to USB device // has been tested manually since currently it is the only place it has been used. - QString cp = "cp"; + QString cmd = "cp"; QStringList arguments; arguments << "-r" << vSource << vDestination; - int result = QProcess::execute(cp, arguments); + int result = QProcess::execute(cmd, arguments); return result; } -// coco end +// disabled coco end /*! + * \brief FileHandler::moveFolder + * \details Moves the source folder to destination. + * \param vFolder - The folder to be removed + * \return True on successful execution. + * \note This method uses the Linux "mv vSource vDestination" command + * Not a Thread-Safe. + */ +int FileHandler::moveFolder(const QString &vSource, const QString &vDestination) +{ + // disabled coco begin validated: This needs user interaction to check the file system + // has been tested manually since currently it is the only place it has been used. + QString cmd = "mv"; + QStringList arguments; + arguments << vSource << vDestination; + int result = QProcess::execute(cmd, arguments); + return result; +} +// disabled coco end + +/*! + * \brief FileHandler::removeFolder + * \details Removes the entire folder. + * \param vFolder - The folder + * \return True on successful execution. + * \note This method uses the Linux "rm -frd vSource vDestination" command + * <<<<< DANGEROUS BE CAREFUL >>>>> + * Not a Thread-Safe. + */ +int FileHandler::removeFolder(const QString &vFolder) +{ + // disabled coco begin validated: This needs user interaction to check the file system + // has been tested manually since currently it is the only place it has been used. + QString cmd = "rm"; + QStringList arguments; + arguments << "-frd" << vFolder; + int result = QProcess::execute(cmd, arguments); + return result; +} +// disabled coco end + +/*! * \brief FileHandler::removeFiles * \details * \param vFolder @@ -147,30 +217,30 @@ for (const auto &info : infoList) { QDateTime fileTime = info.lastModified(); QString fileName = info.absoluteFilePath(); - // coco begin validated : This has been manually tested since requires to change in file system to reproduce the error. + // disabled coco begin validated : This has been manually tested since requires to change in file system to reproduce the error. if (fileTime.isValid()) { if (fileTime.date() <= vDateOlderThan) { if (QFile::remove(fileName)) { ++countRemoved; errOut(QString("%1 File(s) %2 removed").arg(countRemoved).arg(fileName)); } else { - errOut(QString("Can't delete file : ") + fileName); + errOut(QString("Cannot delete file : ") + fileName); } } } else { - errOut(QString("Can't get last modified date of file : ") + fileName); + errOut(QString("Cannot get last modified date of file : ") + fileName); } - // coco end + // disabled coco end } } return countRemoved; } /*! * \brief FileHandler::makeFolder - * \details Create the folder vFolder if it doesn't exist. + * \details Create the folder vFolder if it does not exist. * \param vFolder - the folder to create * \return true on successful creation */ @@ -179,8 +249,8 @@ QDir dir(vFolder); if ( ! dir.exists(vFolder) ) { if ( ! dir.mkpath(vFolder) ) { - QString msg = "Can't create folder " + vFolder; - // here can't use LOG_XXXX because if the folder can't be created then the log can't be written. + QString msg = "Cannot create folder " + vFolder; + // here cannot use LOG_XXXX because if the folder cannot be created then the log cannot be written. errOut(msg); return false; } @@ -196,15 +266,19 @@ */ bool FileHandler::isMounted(const QString &vPath, bool *vIsReadOnly) { - // coco begin validated: Needed User Interaction to make the device not ready so tested manually bool mounted = false; // removing the extra '/' from the vPath if there is to be able to compare to the root path of the storage QString path = vPath.trimmed(); if (path.isEmpty()) return false; int lastIndex = path.size() - 1; if (path.at(lastIndex) == "/") path.remove(lastIndex, 1); // check to see if the path in the list of mounted rootPaths - foreach (const QStorageInfo &storage, QStorageInfo::mountedVolumes()) { + /// DEBUG: qDebug() << " +++++ " << QStorageInfo::mountedVolumes(); + // FIXME : This function blocks the Device controller thread + // It has been observed during the USB plug test and getting the drive info(space) that + // immediately after the mount this function laggs to get the information for about 2-5 sec. + auto mountedVolumes = QStorageInfo::mountedVolumes(); + foreach (const QStorageInfo &storage, mountedVolumes) { if (storage.isValid() && storage.isReady()) { if ( storage.rootPath() == path ) { if (vIsReadOnly) *vIsReadOnly = storage.isReadOnly(); @@ -215,4 +289,164 @@ } return mounted; } -// coco end + +/*! + * \brief FileHandler::tmpUsable + * \details Checks if the temp folder is available for read and write to file and directory + * \note This function will only chek the temp folder usability once + * and next call will just return the first result, + * assuming that the temp file situation is not going to change and is stable. + * \return true on success + */ +bool FileHandler::tmpUsable() +{ + static bool ok = false; + static bool tested = false; + if ( tested ) return ok; + + QString tmp = Storage::Standard_tmp; + QString tmpTestFolder = tmp + "tmp_test_folder" ; + QString tmpTestFile = tmp + "tmp_test_file" ; + QString tmpTestContent = "tmp_test_content"; + QDir dir(tmp); + if ( ! dir.exists() ) { ok = false; goto lOut; } + if ( ! FileHandler::makeFolder (tmpTestFolder )) { ok = false; goto lOut; } + if ( ! FileHandler::write (tmpTestFile , tmpTestContent )) { ok = false; goto lOut; } + if ( ! FileHandler::read (tmpTestFolder, tmpTestContent )) { ok = false; goto lOut; } +lOut: + tested = true; + return ok; +} + +/*! + * \brief FileHandler::find + * \details The function to find files. + * It mainly been implemented to find the files are using some amount of total storage in the vPath by percentage, + * and are the oldest. + * \param vPath - the path to search for the files + * \param vNameFilters - the files filter to search for. + * \param vRetainPercent - It means how many percentage of the space these file are able to retain. + * e.g. 90% retains means older files which use 10% of storage will be listed. + * e.g. 80% retains means older files which use 20% of storage will be listed. + * e.g. 0% retains means no file shall be retained, therefore all the files listed. + * e.g. 100% retains means all file shall be retained, therefore the list shall be empty. + * By default set t0 0 so do not have any size constraint + * \return list of the files found by their information. + * if vRetainPercent is used then it contains list of the file(s) to be removed. + */ +QFileInfoList FileHandler::find(const QString &vPath, QStringList vNameFilters, quint8 vRetainPercent) { + // disabled coco begin validated: Manually tested. Needs to fill up the storage to test some functionalities like vRetainPercent + + QFileInfoList fileInfoList; + // if all the files need to retain then no file shall be listed in the remove list. + if ( vRetainPercent == 100 ) return fileInfoList; + + // if the path is incorrect return with empty list + QDir dir(vPath); + if (!dir.exists()) return fileInfoList; + + // get the storage total + QStorageInfo storage(dir); + quint64 totalSizeStorage = storage.bytesTotal(); + + // get list of all the files in the path by the filter + QFileInfoList fileInfoListAll = find(vPath, vNameFilters); + + // if there is no file in the path with that filter then return empty list. + if (fileInfoListAll.count() == 0) return fileInfoList; + + // if vRetainPercent is 0 means all needs to be removed. + if (vRetainPercent == 0) return fileInfoListAll; + + // get the files total + quint64 totalSizeFiles = totalSize(fileInfoListAll); + + // the total size that all the + quint64 totalSizeRetain = totalSizeStorage * (vRetainPercent / 100.0); + + // if already totalSizeFiles <= totalSizeRetain, do not go any further and return empty list. + if (totalSizeFiles <= totalSizeRetain) return fileInfoList; + + // gets each file size from oldest to newest + // checks if the total files size subtracted by the current file size will be less that limit + // if it is breaks + // else adds the file to the list and continues. + // DEBUG: qDebug() << "%" << totalSizeStorage << totalSizeFiles << totalSizeRetain << totalSizeFiles - totalSizeRetain << vRetainPercent; + quint64 totalSizeRemoved = 0; Q_UNUSED(totalSizeRemoved); + for (auto it = fileInfoListAll.crbegin(); it != fileInfoListAll.crend(); ++it) { + // (totalSizeFiles <= totalSizeRetain) has been checked above and didn't return; , + // so at least one file should be checked and then check again in the loop. + quint64 size = it->size(); + totalSizeRemoved += size; + totalSizeFiles -= size; + fileInfoList += *it; + /// DEBUG: since it has been manually tested this will help next time for test. + /// debugging the find function + // qDebug() << QString("%1 , %2 , %3 , %4 , %5") + // .arg(totalSizeFiles , 12) + // .arg(size , 12) + // .arg(totalSizeRemoved , 12) + // .arg(it->lastModified().toString("yyyy-MM-dd-HH:mm")) + // .arg(it->fileName()) + // ; + if (totalSizeFiles <= totalSizeRetain) break; + } + /// DEBUG: since it has been manually tested this will help next time for test. + /// the total size & count removed. + // qDebug() << QString("%1 , %2") + // .arg(totalSizeRemoved , 12) + // .arg(fileInfoList.count(), 3) + // ; + return fileInfoList; +} +// disabled coco end + +/*! + * \brief FileHandler::find + * \details The function to find files. + * \param vPath - the path to search for the files + * \param vNameFilters - the files filter to search for. + * \return list of the files found by their information. + * if vRetainPercent is used then it contains list of the file(s) to be removed. + */ +QFileInfoList FileHandler::find(const QString &vPath, QStringList vNameFilters) { + // disabled coco begin validated: Needs to manually create specific folder with specific files to check the functionality + // manually tested + QFileInfoList fileInfoList; + QDir dir(vPath); + if (!dir.exists()) return fileInfoList; + fileInfoList = dir.entryInfoList( + vNameFilters, + QDir::NoDotAndDotDot | QDir::Files, + // the sorting may require to change from QDir::Time to QDir::Name + // since the birthTime always returns invalid + // and as part of our log naming we have the birthTime in file name. + // be careful that this function is being used in cases other than only just logging. + QDir::Time + ); + return fileInfoList; +} +// disabled coco end + +quint64 FileHandler::totalSize(const QFileInfoList &vFileInfoList) { + // disabled coco begin validated: Manually tested. requires list of files on file system to test and requires manual specific files for test. + quint64 total = 0; + for (auto it = vFileInfoList.crbegin(); it != vFileInfoList.crend(); ++it) { + total += it->size(); + } + return total; +} +// disabled coco end + +/*! + * \brief FileHandler::subFolders + * \details Look for the sub-folder in folder vFolder. + * \param vFolder - the folder to search for the sub folders. + * \return list of the sub-folders in QString + */ +QStringList FileHandler::subFolders(const QString &vFolder) +{ + QDir dir(vFolder); + dir.setFilter(QDir::NoDotAndDotDot | QDir::Dirs | QDir::Hidden | QDir::Readable); + return dir.entryList(); +}