refs #619, allow exclude dirs in file utils searches for newest files

src/blackmisc/fileutils.cpp

@@ -9,7 +9,7 @@
 
 #include "blackmisc/worker.h"
 #include "fileutils.h"
-
+#include "project.h"
 #include <QFile>
 #include <QCoreApplication>
 
@@ -72,24 +72,75 @@ namespace BlackMisc
 
             QDir originDir(sourceFileInfo.absoluteFilePath());
             auto fileNames = originDir.entryList(QDir::Files | QDir::Dirs | QDir::NoDotAndDotDot | QDir::Hidden | QDir::System);
-            for (const QString &fileName: fileNames)
+            for (const QString &fileName : fileNames)
             {
                 if (!copyRecursively(originDir.absoluteFilePath(fileName), targetDir.absoluteFilePath(fileName)))
+                {
                     return false;
                 }
             }
+        }
         else
         {
             if (!QFile::copy(sourceDir, destinationDir))
+            {
                 return false;
             }
+        }
 
         return true;
     }
 
-    QString CFileUtils::findFirstFile(const QDir &dir, bool recursive, const QString &wildcard, std::function<bool(const QFileInfo &)> predicate)
+    QString CFileUtils::normalizeFilePathToQtStandard(const QString &filePath)
     {
-        QFileInfoList result = dir.entryInfoList({ wildcard }, QDir::Files);
+        if (filePath.isEmpty()) { return ""; }
+        QString n(filePath);
+        n = n.replace('\\', '/').replace("//", "/");
+        return n;
+    }
+
+    Qt::CaseSensitivity CFileUtils::osFileNameCaseSensitivity()
+    {
+        return CProject::isRunningOnWindowsNtPlatform() ? Qt::CaseInsensitive : Qt::CaseSensitive;
+    }
+
+    bool CFileUtils::matchesExcludeDirectory(const QString &directoryPath, const QString &excludeDirectory, Qt::CaseSensitivity cs)
+    {
+        if (directoryPath.isEmpty() || excludeDirectory.isEmpty()) { return false; }
+        const QString ed(normalizeFilePathToQtStandard(excludeDirectory));
+        return directoryPath.contains(ed, cs);
+    }
+
+    bool CFileUtils::isExcludedDirectory(const QDir &directory, const QStringList &excludeDirectories, Qt::CaseSensitivity cs)
+    {
+        if (excludeDirectories.isEmpty()) { return false; }
+        const QString d = directory.absolutePath();
+        return isExcludedDirectory(d, excludeDirectories, cs);
+    }
+
+    bool CFileUtils::isExcludedDirectory(const QFileInfo &fileInfo, const QStringList &excludeDirectories, Qt::CaseSensitivity cs)
+    {
+        if (excludeDirectories.isEmpty()) { return false; }
+        return isExcludedDirectory(fileInfo.absoluteDir(), excludeDirectories, cs);
+    }
+
+    bool CFileUtils::isExcludedDirectory(const QString &directoryPath, const QStringList &excludeDirectories, Qt::CaseSensitivity cs)
+    {
+        if (excludeDirectories.isEmpty()) { return false; }
+        for (const QString &ex : excludeDirectories)
+        {
+            if (matchesExcludeDirectory(directoryPath, ex, cs))
+            {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    QString CFileUtils::findFirstFile(const QDir &dir, bool recursive, const QStringList &nameFilters, const QStringList &excludeDirectories, std::function<bool(const QFileInfo &)> predicate)
+    {
+        if (isExcludedDirectory(dir, excludeDirectories)) { return QString(); }
+        const QFileInfoList result = dir.entryInfoList(nameFilters, QDir::Files);
         if (predicate)
         {
             auto it = std::find_if(result.cbegin(), result.cend(), predicate);
@@ -103,31 +154,33 @@ namespace BlackMisc
         {
             for (const auto &subdir : dir.entryInfoList(QDir::Dirs | QDir::NoDotAndDotDot))
             {
-                QString first = findFirstFile(subdir.filePath(), true, wildcard, predicate);
+                if (isExcludedDirectory(subdir, excludeDirectories)) { continue; }
+                const QString first = findFirstFile(subdir.filePath(), true, nameFilters, excludeDirectories, predicate);
                 if (! first.isEmpty()) { return first; }
             }
         }
         return {};
     }
 
-    bool CFileUtils::containsFile(const QDir &dir, bool recursive, const QString &wildcard, std::function<bool(const QFileInfo &)> predicate)
+    bool CFileUtils::containsFile(const QDir &dir, bool recursive, const QStringList &nameFilters, const QStringList &excludeDirectories, std::function<bool(const QFileInfo &)> predicate)
     {
-        return ! findFirstFile(dir, recursive, wildcard, predicate).isEmpty();
+        return ! findFirstFile(dir, recursive, nameFilters, excludeDirectories, predicate).isEmpty();
     }
 
-    QString CFileUtils::findFirstNewerThan(const QDateTime &time, const QDir &dir, bool recursive, const QString &wildcard)
+    QString CFileUtils::findFirstNewerThan(const QDateTime &time, const QDir &dir, bool recursive, const QStringList &nameFilters, const QStringList &excludeDirectories)
     {
-        return findFirstFile(dir, recursive, wildcard, [time](const QFileInfo &fi) { return fi.lastModified() > time; });
+        return findFirstFile(dir, recursive, nameFilters, excludeDirectories, [time](const QFileInfo & fi) { return fi.lastModified() > time; });
     }
 
-    bool CFileUtils::containsFileNewerThan(const QDateTime &time, const QDir &dir, bool recursive, const QString &wildcard)
+    bool CFileUtils::containsFileNewerThan(const QDateTime &time, const QDir &dir, bool recursive, const QStringList &nameFilters, const QStringList &excludeDirectories)
     {
-        return ! findFirstNewerThan(time, dir, recursive, wildcard).isEmpty();
+        return ! findFirstNewerThan(time, dir, recursive, nameFilters, excludeDirectories).isEmpty();
     }
 
-    QFileInfoList CFileUtils::enumerateFiles(const QDir &dir, bool recursive, const QString &wildcard, std::function<bool(const QFileInfo &)> predicate)
+    QFileInfoList CFileUtils::enumerateFiles(const QDir &dir, bool recursive, const QStringList &nameFilters, const QStringList &excludeDirectories, std::function<bool(const QFileInfo &)> predicate)
     {
-        QFileInfoList result = dir.entryInfoList({ wildcard }, QDir::Files);
+        if (isExcludedDirectory(dir, excludeDirectories)) { return QFileInfoList(); }
+        QFileInfoList result = dir.entryInfoList(nameFilters, QDir::Files);
         if (predicate)
         {
             result.erase(std::remove_if(result.begin(), result.end(), std::not1(predicate)), result.end());
@@ -136,18 +189,20 @@ namespace BlackMisc
         {
             for (const auto &subdir : dir.entryInfoList(QDir::Dirs | QDir::NoDotAndDotDot))
             {
-                result += enumerateFiles(subdir.filePath(), true, wildcard, predicate);
+                if (isExcludedDirectory(subdir, excludeDirectories)) { continue; }
+                result += enumerateFiles(subdir.filePath(), true, nameFilters, excludeDirectories, predicate);
             }
         }
         return result;
     }
 
-    QString CFileUtils::findNewestFile(const QDir &dir, bool recursive, const QString &wildcard)
+    QString CFileUtils::findNewestFile(const QDir &dir, bool recursive, const QStringList &nameFilters, const QStringList &excludeDirectories)
     {
-        const QFileInfoList files = enumerateFiles(dir, recursive, wildcard);
+        if (isExcludedDirectory(dir, excludeDirectories)) { return QString(); }
+        const QFileInfoList files = enumerateFiles(dir, recursive, nameFilters, excludeDirectories);
         if (files.isEmpty()) { return {}; }
 
-        auto it = std::max_element(files.cbegin(), files.cend(), [](const QFileInfo &a, const QFileInfo &b)
+        auto it = std::max_element(files.cbegin(), files.cend(), [](const QFileInfo & a, const QFileInfo & b)
         {
             return a.lastModified() < b.lastModified();
         });

src/blackmisc/fileutils.h

@@ -45,23 +45,41 @@ namespace BlackMisc
         //! If it is a file, just copies the file.
         static bool copyRecursively(const QString &sourceDir, const QString &destinationDir);
 
+        //! Normalize file path to Qt standard, e.g by turning \ to /
+        static QString normalizeFilePathToQtStandard(const QString &filePath);
+
+        //! Case sensitivity for current OS
+        static Qt::CaseSensitivity osFileNameCaseSensitivity();
+
+        //! Is directory path matching the exclude path?
+        static bool matchesExcludeDirectory(const QString &directoryPath, const QString &excludeDirectory, Qt::CaseSensitivity cs = osFileNameCaseSensitivity());
+
+        //! Directory to be excluded?
+        static bool isExcludedDirectory(const QDir &directory, const QStringList &excludeDirectories, Qt::CaseSensitivity cs = osFileNameCaseSensitivity());
+
+        //! Directory to be excluded?
+        static bool isExcludedDirectory(const QFileInfo &fileInfo, const QStringList &excludeDirectories, Qt::CaseSensitivity cs = osFileNameCaseSensitivity());
+
+        //! Directory to be excluded?
+        static bool isExcludedDirectory(const QString &directoryPath, const QStringList &excludeDirectories, Qt::CaseSensitivity cs = osFileNameCaseSensitivity());
+
         //! Returns path to first file in dir which matches the optional wildcard and predicate, or empty string.
-        static QString findFirstFile(const QDir &dir, bool recursive, const QString &wildcard = {}, std::function<bool(const QFileInfo &)> predicate = {});
+        static QString findFirstFile(const QDir &dir, bool recursive, const QStringList &nameFilters = {}, const QStringList &excludeDirectories = {}, std::function<bool(const QFileInfo &)> predicate = {});
 
         //! True if there exists a file in dir which matches the optional wildcard and predicate.
-        static bool containsFile(const QDir &dir, bool recursive, const QString &wildcard = {}, std::function<bool(const QFileInfo &)> predicate = {});
+        static bool containsFile(const QDir &dir, bool recursive, const QStringList &nameFilters = {}, const QStringList &excludeDirectories = {}, std::function<bool(const QFileInfo &)> predicate = {});
 
         //! Returns path to first file in dir newer than the given time, optionally matching a wildcard, or empty string.
-        static QString findFirstNewerThan(const QDateTime &time, const QDir &dir, bool recursive, const QString &wildcard = {});
+        static QString findFirstNewerThan(const QDateTime &time, const QDir &dir, bool recursive, const QStringList &nameFilters = {}, const QStringList &excludeDirectories = {});
 
         //! True if there exists a file in dir newer than the given time, optionally matching a wildcard.
-        static bool containsFileNewerThan(const QDateTime &time, const QDir &dir, bool recursive, const QString &wildcard = {});
+        static bool containsFileNewerThan(const QDateTime &time, const QDir &dir, bool recursive, const QStringList &nameFilters = {}, const QStringList &excludeDirectories = {});
 
         //! Returns list of all files in dir, optionally matching a wildcard and predicate.
-        static QFileInfoList enumerateFiles(const QDir &dir, bool recursive, const QString &wildcard = {}, std::function<bool(const QFileInfo &)> predicate = {});
+        static QFileInfoList enumerateFiles(const QDir &dir, bool recursive, const QStringList &nameFilters = {}, const QStringList &excludeDirectories = {}, std::function<bool(const QFileInfo &)> predicate = {});
 
         //! Returns path to the newest file in dir, optionally matching a wildcard, or empty string.
-        static QString findNewestFile(const QDir &dir, bool recursive, const QString &wildcard = {});
+        static QString findNewestFile(const QDir &dir, bool recursive, const QStringList &nameFilters = {}, const QStringList &excludeDirectories = {});
     };
 } // ns
 

src/blackmisc/simulation/fscommon/aircraftcfgparser.cpp

@@ -148,8 +148,7 @@ namespace BlackMisc
             {
                 const QDateTime cacheTs(getCacheTimestamp());
                 if (!cacheTs.isValid()) { return true; }
-                //! \todo KB we cannot use the exclude dirs, a minor disadvantege. Also wonder if it was better to parse a QStringList ofr wildcard
+                return CFileUtils::containsFileNewerThan(cacheTs, this->getRootDirectory(), true, { fileFilter() }, this->m_excludedDirectories);
-                return CFileUtils::containsFileNewerThan(cacheTs, this->getRootDirectory(), true, "*.cfg");
             }
 
             bool CAircraftCfgParser::hasCachedData() const
@@ -223,19 +222,15 @@ namespace BlackMisc
                 if (m_cancelLoading) { return CAircraftCfgEntriesList(); }
 
                 // excluded?
-                for (const auto &excludeDir : excludeDirectories)
+                if (CFileUtils::isExcludedDirectory(directory, excludeDirectories))
-                {
-                    if (m_cancelLoading) { return CAircraftCfgEntriesList(); }
-                    if (directory.contains(excludeDir, Qt::CaseInsensitive))
                 {
                     CLogMessage(this).debug() << "Skipping directory " << directory;
                     *ok = true;
                     return CAircraftCfgEntriesList();
                 }
-                }
 
                 // set directory with name filters, get aircraft.cfg and sub directories
-                QDir dir(directory, "aircraft.cfg", QDir::Name, QDir::Files | QDir::AllDirs);
+                QDir dir(directory, fileFilter(), QDir::Name, QDir::Files | QDir::AllDirs | QDir::NoDotAndDotDot);
                 if (!dir.exists())
                 {
                     *ok = true;
@@ -446,6 +441,12 @@ namespace BlackMisc
                 return content;
             }
 
+            const QString &CAircraftCfgParser::fileFilter()
+            {
+                static const QString f("aircraft.cfg");
+                return f;
+            }
+
         } // namespace
     } // namespace
 } // namespace

src/blackmisc/simulation/fscommon/aircraftcfgparser.h

@@ -94,10 +94,12 @@ namespace BlackMisc
                 CAircraftCfgEntriesList m_parsedCfgEntriesList; //!< parsed entries
                 QPointer<BlackMisc::CWorker> m_parserWorker;    //!< worker will destroy itself, so weak pointer
 
-                //! \todo KB/MS Is there nothing better than having 3 cache members
+                //! \todo KB/MS Is there nothing better than having 3 cache members?
                 BlackMisc::CData<BlackMisc::Simulation::Data::ModelCacheFsx> m_modelCacheFsx {this};  //!< FSX cache
                 BlackMisc::CData<BlackMisc::Simulation::Data::ModelCacheFs9> m_modelCacheFs9 {this};  //!< Fs9 cache
                 BlackMisc::CData<BlackMisc::Simulation::Data::ModelCacheP3D> m_modelCacheP3D {this};  //!< P3D cache
+
+                static const QString &fileFilter();
             };
         } // namespace
     } // namespace

src/blackmisc/simulation/xplane/aircraftmodelloaderxplane.cpp

@@ -32,9 +32,9 @@ namespace BlackMisc
     {
         namespace XPlane
         {
-
             static void normalizePath(QString &path)
             {
+                //! \todo KB consider CFileUtil::normalizeFilePathToQtStandard ??
                 for (auto &e : path)
                 {
                     if (e == '/' || e == ':' || e == '\\')
@@ -103,8 +103,7 @@ namespace BlackMisc
             {
                 const QDateTime cacheTs(getCacheTimestamp());
                 if (!cacheTs.isValid()) { return true; }
-                //! \todo KB we cannot use the exclude dirs, a minor disadvantege. Also wonder if it was better to parse a QStringList ofr wildcard
+                return CFileUtils::containsFileNewerThan(cacheTs, this->getRootDirectory(), true, {fileFilterCsl(), fileFilterFlyable()},  this->m_excludedDirectories);
-                return CFileUtils::containsFileNewerThan(cacheTs, this->getRootDirectory(), true, "xsb_aircraft.txt");
             }
 
             bool CAircraftModelLoaderXPlane::hasCachedData() const
@@ -163,12 +162,13 @@ namespace BlackMisc
                 Q_UNUSED(excludeDirectories);
                 if (rootDirectory.isEmpty()) { return {}; }
 
-                QDir searchPath(rootDirectory, "*.acf");
+                QDir searchPath(rootDirectory, fileFilterFlyable());
                 QDirIterator aircraftIt(searchPath, QDirIterator::Subdirectories | QDirIterator::FollowSymlinks);
 
                 CAircraftModelList installedModels;
                 while (aircraftIt.hasNext())
                 {
+                    //! \todo KB I would consider exclude dirs here CFileUtils::matchesExcludeDirectory()
                     aircraftIt.next();
 
                     // <dirname> <filename> for the default model and <dirname> <filename> <texturedir> for the liveries
@@ -184,6 +184,7 @@ namespace BlackMisc
                     file.open(QIODevice::ReadOnly | QIODevice::Text);
 
                     QTextStream ts(&file);
+                    //! \todo Do you rely on case sensitive parsing, mabe case insensitive would be better?
                     if (ts.readLine() == "I" && ts.readLine().contains("version") && ts.readLine() == "ACF")
                     {
                         while (!ts.atEnd())
@@ -223,11 +224,12 @@ namespace BlackMisc
 
                 m_cslPackages.clear();
 
-                QDir searchPath(rootDirectory, "xsb_aircraft.txt");
+                QDir searchPath(rootDirectory, fileFilterCsl());
                 QDirIterator it(searchPath, QDirIterator::Subdirectories);
                 while (it.hasNext())
                 {
                     QString packageFile = it.next();
+                    //! \todo KB I would consider exclude dirs here CFileUtils::matchesExcludeDirectory()
 
                     QString packageFilePath = it.fileInfo().absolutePath();
                     QFile file(packageFile);
@@ -637,6 +639,18 @@ namespace BlackMisc
                 }
             }
 
+            const QString &CAircraftModelLoaderXPlane::fileFilterFlyable()
+            {
+                static const QString f("*.acf");
+                return f;
+            }
+
+            const QString &CAircraftModelLoaderXPlane::fileFilterCsl()
+            {
+                static const QString f("xsb_aircraft.txt");
+                return f;
+            }
+
         } // namespace
     } // namespace
 } // namespace

src/blackmisc/simulation/xplane/aircraftmodelloaderxplane.h

@@ -114,6 +114,9 @@ namespace BlackMisc
                 QPointer<BlackMisc::CWorker> m_parserWorker;    //!< worker will destroy itself, so weak pointer
                 QVector<CSLPackage> m_cslPackages;              //!< Parsed Packages. No lock required since accessed only from one thread
                 BlackMisc::Simulation::CAircraftModelList m_installedModels;
+
+                static const QString &fileFilterFlyable();
+                static const QString &fileFilterCsl();
             };
         } // namespace
     } // namespace