Added caching support cmd line module xml descriptions.

Applications/ctkCommandLineModuleExplorer/ctkCmdLineModuleExplorerMainWindow.cpp

@@ -36,14 +36,15 @@
 
 #include <ctkSettingsDialog.h>
 
+#include <QDesktopServices>
 #include <QDebug>
 
 
 ctkCLModuleExplorerMainWindow::ctkCLModuleExplorerMainWindow(QWidget *parent) :
   QMainWindow(parent),
   ui(new Ui::ctkCmdLineModuleExplorerMainWindow),
-  tabList(NULL),
   defaultModuleFrontendFactory(NULL),
+  moduleManager(ctkCmdLineModuleManager::STRICT_VALIDATION, QDesktopServices::storageLocation(QDesktopServices::CacheLocation)),
   directoryWatcher(&moduleManager)
 {
   ui->setupUi(this);

Applications/ctkCommandLineModuleExplorer/ctkCommandLineModuleExplorerMain.cpp

@@ -41,6 +41,8 @@
 int main(int argc, char** argv)
 {
   QApplication myApp(argc, argv);
+  myApp.setOrganizationName("CommonTK");
+  myApp.setApplicationName("CommandLineModuleExplorer");
 
   ctkCommandLineParser cmdLineParser;
   cmdLineParser.setArgumentPrefix("--", "-");

Libs/CommandLineModules/Backend/FunctionPointer/ctkCmdLineModuleBackendFunctionPointer.cpp

@@ -173,6 +173,12 @@ QList<QString> ctkCmdLineModuleBackendFunctionPointer::schemes() const
   return supportedSchemes;
 }
 
+qint64 ctkCmdLineModuleBackendFunctionPointer::timeStamp(const QUrl &location) const
+{
+  Q_UNUSED(location)
+  return 0;
+}
+
 QByteArray ctkCmdLineModuleBackendFunctionPointer::rawXmlDescription(const QUrl& location)
 {
   if (!d->UrlToFpDescription.contains(location)) return QByteArray();

Libs/CommandLineModules/Backend/FunctionPointer/ctkCmdLineModuleBackendFunctionPointer.h

@@ -99,6 +99,8 @@ public:
 
   virtual QList<QString> schemes() const;
 
+  virtual qint64 timeStamp(const QUrl &location) const;
+
   virtual QByteArray rawXmlDescription(const QUrl& location);
 
   virtual ctkCmdLineModuleFuture run(ctkCmdLineModuleFrontend *frontend);

Libs/CommandLineModules/Backend/LocalProcess/ctkCmdLineModuleBackendLocalProcess.cpp

@@ -30,6 +30,8 @@
 #include "ctkCmdLineModuleReference.h"
 #include "ctkCmdLineModuleRunException.h"
 
+#include "ctkUtils.h"
+
 #include <QProcess>
 #include <QUrl>
 
@@ -126,6 +128,17 @@ QList<QString> ctkCmdLineModuleBackendLocalProcess::schemes() const
   return supportedSchemes;
 }
 
+qint64 ctkCmdLineModuleBackendLocalProcess::timeStamp(const QUrl &location) const
+{
+  QFileInfo fileInfo(location.toLocalFile());
+  if (fileInfo.exists())
+  {
+    QDateTime dateTime = fileInfo.lastModified();
+    return ctk::msecsTo(QDateTime::fromTime_t(0), dateTime);
+  }
+  return 0;
+}
+
 QByteArray ctkCmdLineModuleBackendLocalProcess::rawXmlDescription(const QUrl &location)
 {
   QProcess process;

Libs/CommandLineModules/Backend/LocalProcess/ctkCmdLineModuleBackendLocalProcess.h

@@ -43,6 +43,8 @@ public:
 
   virtual QList<QString> schemes() const;
 
+  virtual qint64 timeStamp(const QUrl &location) const;
+
   virtual QByteArray rawXmlDescription(const QUrl& location);
 
   virtual ctkCmdLineModuleFuture run(ctkCmdLineModuleFrontend *frontend);

Libs/CommandLineModules/Core/CMakeLists.txt

@@ -15,6 +15,8 @@ set(KIT_export_directive "CTK_CMDLINEMODULECORE_EXPORT")
 # Source files
 set(KIT_SRCS
   ctkCmdLineModuleBackend.cpp
+  ctkCmdLineModuleCache.cpp
+  ctkCmdLineModuleCache_p.h
   ctkCmdLineModuleDefaultPathBuilder.cpp
   ctkCmdLineModuleDescription.cpp
   ctkCmdLineModuleDescription_p.h

Libs/CommandLineModules/Core/ctkCmdLineModuleBackend.h

@@ -40,6 +40,8 @@ struct CTK_CMDLINEMODULECORE_EXPORT ctkCmdLineModuleBackend
 
   virtual QList<QString> schemes() const = 0;
 
+  virtual qint64 timeStamp(const QUrl& location) const = 0;
+
   /**
    * @brief Get the XML parameter description from the given location.
    * @param location The location URL specifying the module.

Libs/CommandLineModules/Core/ctkCmdLineModuleCache.cpp

@@ -0,0 +1,174 @@
+/*=============================================================================
+
+  Library: CTK
+
+  Copyright (c) German Cancer Research Center,
+    Division of Medical and Biological Informatics
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+
+=============================================================================*/
+
+#include "ctkCmdLineModuleCache_p.h"
+
+#include <QUrl>
+#include <QFile>
+#include <QDirIterator>
+#include <QTextStream>
+#include <QMutex>
+
+struct ctkCmdLineModuleCachePrivate
+{
+  QString CacheDir;
+
+  QHash<QUrl, qint64> LocationToTimeStamp;
+  QHash<QUrl, QByteArray> LocationToXmlDescription;
+
+  QMutex Mutex;
+
+  void LoadTimeStamps()
+  {
+    QDirIterator dirIter(this->CacheDir, QStringList() << "*.timestamp", QDir::Files | QDir::Readable);
+    while(dirIter.hasNext())
+    {
+      QFile timestampFile(dirIter.next());
+      timestampFile.open(QIODevice::ReadOnly);
+      QUrl url = QUrl(timestampFile.readLine().trimmed().data());
+      QByteArray timestamp = timestampFile.readLine();
+      bool ok = false;
+      qint64 ts = timestamp.toLongLong(&ok);
+      if (ok && !url.isEmpty())
+      {
+        this->LocationToTimeStamp[url] = ts;
+      }
+    }
+  }
+
+  QString timeStampFileName(const QUrl& moduleLocation) const
+  {
+    return this->CacheDir + "/" + QString::number(qHash(moduleLocation)) + ".timestamp";
+  }
+
+  QString xmlFileName(const QUrl& moduleLocation) const
+  {
+    return this->CacheDir + "/" + QString::number(qHash(moduleLocation)) + ".xml";
+  }
+};
+
+ctkCmdLineModuleCache::ctkCmdLineModuleCache(const QString& cacheDir)
+  : d(new ctkCmdLineModuleCachePrivate)
+{
+  d->CacheDir = cacheDir;
+  d->LoadTimeStamps();
+}
+
+ctkCmdLineModuleCache::~ctkCmdLineModuleCache()
+{
+}
+
+QString ctkCmdLineModuleCache::cacheDir() const
+{
+  QMutexLocker lock(&d->Mutex);
+  return d->CacheDir;
+}
+
+QByteArray ctkCmdLineModuleCache::rawXmlDescription(const QUrl& moduleLocation) const
+{
+  QMutexLocker lock(&d->Mutex);
+
+  if (d->LocationToXmlDescription.contains(moduleLocation))
+  {
+    return d->LocationToXmlDescription[moduleLocation];
+  }
+  // lazily load the XML description from the file system
+  QByteArray xml;
+  QString a = moduleLocation.toString();
+  QString fn = d->xmlFileName(moduleLocation);
+  QFile xmlFile(d->xmlFileName(moduleLocation));
+  if (xmlFile.exists() && xmlFile.open(QIODevice::ReadOnly))
+  {
+    xml = xmlFile.readAll();
+    xmlFile.close();
+  }
+  d->LocationToXmlDescription[moduleLocation] = xml;
+  return xml;
+}
+
+qint64 ctkCmdLineModuleCache::timeStamp(const QUrl& moduleLocation) const
+{
+  QMutexLocker lock(&d->Mutex);
+  if (d->LocationToTimeStamp.contains(moduleLocation))
+  {
+    return d->LocationToTimeStamp[moduleLocation];
+  }
+  return -1;
+}
+
+void ctkCmdLineModuleCache::cacheXmlDescription(const QUrl& moduleLocation, qint64 timestamp, const QByteArray& xmlDescription)
+{
+  QFile timestampFile(d->timeStampFileName(moduleLocation));
+  QFile xmlFile(d->xmlFileName(moduleLocation));
+  timestampFile.remove();
+  timestampFile.open(QIODevice::WriteOnly);
+
+  QByteArray ba;
+  QTextStream str(&ba);
+  str << moduleLocation.toString() << '\n' << timestamp;
+  str.flush();
+  if (timestampFile.write(ba) == -1)
+  {
+    timestampFile.close();
+    timestampFile.remove();
+    return;
+  }
+  timestampFile.close();
+
+  xmlFile.remove();
+  if (!xmlDescription.isEmpty())
+  {
+    xmlFile.open(QIODevice::WriteOnly);
+    if (xmlFile.write(xmlDescription) == -1)
+    {
+      timestampFile.remove();
+      xmlFile.close();
+      xmlFile.remove();
+      return;
+    }
+  }
+
+  {
+    QMutexLocker lock(&d->Mutex);
+    d->LocationToXmlDescription[moduleLocation] = xmlDescription;
+    d->LocationToTimeStamp[moduleLocation] = timestamp;
+  }
+}
+
+void ctkCmdLineModuleCache::removeCacheEntry(const QUrl& moduleLocation)
+{
+  {
+    QMutexLocker lock(&d->Mutex);
+    d->LocationToTimeStamp.remove(moduleLocation);
+    d->LocationToXmlDescription.remove(moduleLocation);
+  }
+
+  QFile timestampFile(d->timeStampFileName(moduleLocation));
+  if (timestampFile.exists())
+  {
+    timestampFile.remove();
+  }
+  QFile xmlFile(d->xmlFileName(moduleLocation));
+  if (xmlFile.exists())
+  {
+    xmlFile.remove();
+  }
+}

Libs/CommandLineModules/Core/ctkCmdLineModuleCache_p.h

@@ -0,0 +1,53 @@
+/*=============================================================================
+
+  Library: CTK
+
+  Copyright (c) German Cancer Research Center,
+    Division of Medical and Biological Informatics
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+
+=============================================================================*/
+
+#ifndef CTKCMDLINEMODULECACHE_H
+#define CTKCMDLINEMODULECACHE_H
+
+#include <QScopedPointer>
+
+class ctkCmdLineModuleCachePrivate;
+
+class QUrl;
+
+class ctkCmdLineModuleCache
+{
+
+public:
+
+  ctkCmdLineModuleCache(const QString& cacheDir);
+  ~ctkCmdLineModuleCache();
+
+  QString cacheDir() const;
+
+  QByteArray rawXmlDescription(const QUrl& moduleLocation) const;
+  qint64 timeStamp(const QUrl& moduleLocation) const;
+
+  void cacheXmlDescription(const QUrl& moduleLocation, qint64 timestamp, const QByteArray& xmlDescription);
+
+  void removeCacheEntry(const QUrl& moduleLocation);
+
+private:
+
+  QScopedPointer<ctkCmdLineModuleCachePrivate> d;
+};
+
+#endif // CTKCMDLINEMODULECACHE_H

Libs/CommandLineModules/Core/ctkCmdLineModuleManager.cpp

@@ -23,6 +23,7 @@
 
 #include "ctkCmdLineModuleBackend.h"
 #include "ctkCmdLineModuleFrontend.h"
+#include "ctkCmdLineModuleCache_p.h"
 #include "ctkCmdLineModuleFuture.h"
 #include "ctkCmdLineModuleXmlValidator.h"
 #include "ctkCmdLineModuleReference.h"
@@ -30,6 +31,8 @@
 
 #include <ctkException.h>
 
+#include <QFileInfo>
+#include <QDir>
 #include <QStringList>
 #include <QBuffer>
 #include <QUrl>
@@ -41,9 +44,28 @@
 
 struct ctkCmdLineModuleManagerPrivate
 {
-  ctkCmdLineModuleManagerPrivate(ctkCmdLineModuleManager::ValidationMode mode)
+  ctkCmdLineModuleManagerPrivate(ctkCmdLineModuleManager::ValidationMode mode, const QString& cacheDir)
     : ValidationMode(mode)
-  {}
+  {
+    QFileInfo fileInfo(cacheDir);
+    if (!fileInfo.exists())
+    {
+      if (!QDir().mkpath(cacheDir))
+      {
+        qWarning() << "Command line module cache disabled. Directory" << cacheDir << "could not be created.";
+        return;
+      }
+    }
+
+    if (fileInfo.isWritable())
+    {
+      ModuleCache.reset(new ctkCmdLineModuleCache(cacheDir));
+    }
+    else
+    {
+      qWarning() << "Command line module cache disabled. Directory" << cacheDir << "is not writable.";
+    }
+  }
 
   void checkBackends_unlocked(const QUrl& location)
   {
@@ -56,12 +78,13 @@ struct ctkCmdLineModuleManagerPrivate
   QMutex Mutex;
   QHash<QString, ctkCmdLineModuleBackend*> SchemeToBackend;
   QHash<QUrl, ctkCmdLineModuleReference> LocationToRef;
+  QScopedPointer<ctkCmdLineModuleCache> ModuleCache;
 
   const ctkCmdLineModuleManager::ValidationMode ValidationMode;
 };
 
-ctkCmdLineModuleManager::ctkCmdLineModuleManager(ValidationMode validationMode)
-  : d(new ctkCmdLineModuleManagerPrivate(validationMode))
+ctkCmdLineModuleManager::ctkCmdLineModuleManager(ValidationMode validationMode, const QString& cacheDir)
+  : d(new ctkCmdLineModuleManagerPrivate(validationMode, cacheDir))
 {
 }
 
@@ -111,9 +134,43 @@ ctkCmdLineModuleManager::registerModule(const QUrl &location)
     backend = d->SchemeToBackend[location.scheme()];
   }
 
-  xml = backend->rawXmlDescription(location);
+  bool fromCache = false;
+  qint64 newTimeStamp = 0;
+  if (d->ModuleCache)
+  {
+    newTimeStamp = backend->timeStamp(location);
+    if (d->ModuleCache->timeStamp(location) < newTimeStamp)
+    {
+      // newly fetch the XML description
+      try
+      {
+        xml = backend->rawXmlDescription(location);
+      }
+      catch (...)
+      {
+        // cache the failed attempt
+        d->ModuleCache->cacheXmlDescription(location, newTimeStamp, QByteArray());
+        throw;
+      }
+    }
+    else
+    {
+      // use the cached XML description
+      xml = d->ModuleCache->rawXmlDescription(location);
+      fromCache = true;
+    }
+  }
+  else
+  {
+    xml = backend->rawXmlDescription(location);
+  }
+
   if (xml.isEmpty())
   {
+    if (!fromCache && d->ModuleCache)
+    {
+      d->ModuleCache->cacheXmlDescription(location, newTimeStamp, QByteArray());
+    }
     throw ctkInvalidArgumentException(QString("No XML output available from ") + location.toString());
   }
 
@@ -131,6 +188,12 @@ ctkCmdLineModuleManager::registerModule(const QUrl &location)
     ctkCmdLineModuleXmlValidator validator(&input);
     if (!validator.validateInput())
     {
+      if (d->ModuleCache)
+      {
+        // validation failed, cache an empty description
+        d->ModuleCache->cacheXmlDescription(location, newTimeStamp, QByteArray());
+      }
+
       if (d->ValidationMode == STRICT_VALIDATION)
       {
         throw ctkInvalidArgumentException(QString("Validating module at %1 failed: %2")
@@ -141,6 +204,22 @@ ctkCmdLineModuleManager::registerModule(const QUrl &location)
         ref.d->XmlValidationErrorString = validator.errorString();
       }
     }
+    else
+    {
+      if (d->ModuleCache && newTimeStamp > 0)
+      {
+        // successfully validated the xml, cache it
+        d->ModuleCache->cacheXmlDescription(location, newTimeStamp, xml);
+      }
+    }
+  }
+  else
+  {
+    if (!fromCache && d->ModuleCache)
+    {
+      // cache it
+      d->ModuleCache->cacheXmlDescription(location, newTimeStamp, xml);
+    }
   }
 
   {
@@ -167,6 +246,10 @@ void ctkCmdLineModuleManager::unregisterModule(const ctkCmdLineModuleReference&
       return;
     }
     d->LocationToRef.remove(ref.location());
+    if (d->ModuleCache)
+    {
+      d->ModuleCache->removeCacheEntry(ref.location());
+    }
   }
   emit moduleUnregistered(ref);
 }

Libs/CommandLineModules/Core/ctkCmdLineModuleManager.h

@@ -56,7 +56,7 @@ public:
     WEAK_VALIDATION
   };
 
-  ctkCmdLineModuleManager(ValidationMode = STRICT_VALIDATION);
+  ctkCmdLineModuleManager(ValidationMode = STRICT_VALIDATION, const QString& cacheDir = QString());
 
   ~ctkCmdLineModuleManager();