Add schema updating to ctkDICOM code

This is based on work by Marco Nolden to do an in-place update
of the database schema.

The new commits here provide a way to detect when a schema
update is needed using a new table in the database.

Also the schema update emits signals as the update goes on.

Also there is now a test and some examples of old schema so
we can verify things are working.

Also there is a uniform method to reset the effiency variables
so they don't accidentally result in missed entries after
removing data or updating the schema.

The app widget and ctkDICOM example application now support updating
the schema as neeed.

Libs/DICOM/Core/Resources/dicom-old-schema.sql

@@ -0,0 +1,61 @@
+-- 
+-- A simple SQLITE3 database schema for modelling locally stored DICOM files 
+-- 
+-- Note: the semicolon at the end is necessary for the simple parser to separate
+--       the statements since the SQlite driver does not handle multiple
+--       commands per QSqlQuery::exec call!
+-- ;
+
+DROP TABLE IF EXISTS 'Images' ;
+DROP TABLE IF EXISTS 'Patients' ;
+DROP TABLE IF EXISTS 'Series' ;
+DROP TABLE IF EXISTS 'Studies' ;
+DROP TABLE IF EXISTS 'Directories' ;
+
+CREATE TABLE 'Images' (
+  'SOPInstanceUID' VARCHAR(64) NOT NULL,
+  'Filename' VARCHAR(1024) NOT NULL ,
+  'SeriesInstanceUID' VARCHAR(64) NOT NULL ,
+  'InsertTimestamp' VARCHAR(20) NOT NULL ,
+  PRIMARY KEY ('SOPInstanceUID') );
+CREATE TABLE 'Patients' (
+  'UID' INTEGER PRIMARY KEY AUTOINCREMENT,
+  'PatientsName' VARCHAR(255) NULL ,
+  'PatientID' VARCHAR(255) NULL ,
+  'PatientsBirthDate' DATE NULL ,
+  'PatientsBirthTime' TIME NULL ,
+  'PatientsSex' varchar(1) NULL ,
+  'PatientsAge' varchar(10) NULL ,
+  'PatientsComments' VARCHAR(255) NULL );
+CREATE TABLE 'Series' (
+  'SeriesInstanceUID' VARCHAR(64) NOT NULL ,
+  'StudyInstanceUID' VARCHAR(64) NOT NULL ,
+  'SeriesNumber' INT NULL ,
+  'SeriesDate' DATE NULL ,
+  'SeriesTime' VARCHAR(20) NULL ,
+  'SeriesDescription' VARCHAR(255) NULL ,
+  'BodyPartExamined' VARCHAR(255) NULL ,
+  'FrameOfReferenceUID' VARCHAR(64) NULL ,
+  'AcquisitionNumber' INT NULL ,
+  'ContrastAgent' VARCHAR(255) NULL ,
+  'ScanningSequence' VARCHAR(45) NULL ,
+  'EchoNumber' INT NULL ,
+  'TemporalPosition' INT NULL ,
+  PRIMARY KEY ('SeriesInstanceUID') );
+CREATE TABLE 'Studies' (
+  'StudyInstanceUID' VARCHAR(64) NOT NULL ,
+  'PatientsUID' INT NOT NULL ,
+  'StudyID' VARCHAR(255) NULL ,
+  'StudyDate' DATE NULL ,
+  'StudyTime' VARCHAR(20) NULL ,
+  'AccessionNumber' VARCHAR(255) NULL ,
+  'ModalitiesInStudy' VARCHAR(255) NULL ,
+  'InstitutionName' VARCHAR(255) NULL ,
+  'ReferringPhysician' VARCHAR(255) NULL ,
+  'PerformingPhysiciansName' VARCHAR(255) NULL ,
+  'StudyDescription' VARCHAR(255) NULL ,
+  PRIMARY KEY ('StudyInstanceUID') );
+
+CREATE TABLE 'Directories' (
+  'Dirname' VARCHAR(1024) ,
+  PRIMARY KEY ('Dirname') );

Libs/DICOM/Core/Resources/dicom-schema.sql

@@ -6,12 +6,16 @@
 --       commands per QSqlQuery::exec call!
 -- ;
 
+DROP TABLE IF EXISTS 'SchemaInfo' ;
 DROP TABLE IF EXISTS 'Images' ;
 DROP TABLE IF EXISTS 'Patients' ;
 DROP TABLE IF EXISTS 'Series' ;
 DROP TABLE IF EXISTS 'Studies' ;
 DROP TABLE IF EXISTS 'Directories' ;
 
+CREATE TABLE 'SchemaInfo' ( 'Version' VARCHAR(1024) NOT NULL );
+INSERT INTO 'SchemaInfo' VALUES('0.5');
+
 CREATE TABLE 'Images' (
   'SOPInstanceUID' VARCHAR(64) NOT NULL,
   'Filename' VARCHAR(1024) NOT NULL ,

Libs/DICOM/Core/Testing/Cpp/CMakeLists.txt

@@ -4,6 +4,7 @@ create_test_sourcelist(Tests ${KIT}CppTests.cpp
   ctkDICOMCoreTest1.cpp
   ctkDICOMDatabaseTest1.cpp
   ctkDICOMDatabaseTest2.cpp
+  ctkDICOMDatabaseTest3.cpp
   ctkDICOMDatasetTest1.cpp
   ctkDICOMIndexerTest1.cpp
   ctkDICOMModelTest1.cpp
@@ -31,6 +32,9 @@ target_link_libraries(${KIT}CppTests ${LIBRARY_NAME})
 # ctkDICOMDatabase
 SIMPLE_TEST(ctkDICOMDatabaseTest1)
 SIMPLE_TEST(ctkDICOMDatabaseTest2 ${CTKData_DIR}/Data/DICOM/MRHEAD/000055.IMA)
+SIMPLE_TEST(ctkDICOMDatabaseTest3
+  ${CMAKE_CURRENT_SOURCE_DIR}/../../Resources/dicom-old-schema.sql
+  )
 SIMPLE_TEST(ctkDICOMDatasetTest1)
 SIMPLE_TEST(ctkDICOMIndexerTest1 )
 

Libs/DICOM/Core/Testing/Cpp/ctkDICOMDatabaseTest3.cpp

@@ -0,0 +1,114 @@
+/*=========================================================================
+
+  Library:   CTK
+
+  Copyright (c) Kitware Inc.
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0.txt
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+
+=========================================================================*/
+
+// Qt includes
+#include <QCoreApplication>
+#include <QDir>
+
+// ctkDICOMCore includes
+#include "ctkDICOMDatabase.h"
+
+// STD includes
+#include <iostream>
+#include <cstdlib>
+
+
+int ctkDICOMDatabaseTest3( int argc, char * argv [] )
+{
+  QCoreApplication app(argc, argv);
+
+  if (argc <= 1)
+    {
+    std::cerr << "Warning, no sql file given. Test stops" << std::endl;
+    std::cerr << "Usage: ctkDICOMDatabaseTest3 <olddumpfile.sql>" << std::endl;
+    return EXIT_FAILURE;
+    }
+
+  QDir databaseDirectory = QDir::temp();
+  databaseDirectory.remove("ctkDICOMDatabase.sql");
+
+  QFileInfo databaseFile(databaseDirectory, QString("database.test"));
+  QString databaseFileName(databaseFile.absoluteFilePath());
+  
+  std::cerr << "Populating database " << databaseFileName.toStdString() << "\n";
+
+  // first, create a database and initialize it with the old schema
+  try
+  {
+    ctkDICOMDatabase myCTK( databaseFileName );
+
+    if (!myCTK.initializeDatabase(argv[1]))
+    {
+      std::cerr << "Error when initializing the data base with: " << argv[1]
+          << " error: " << myCTK.lastError().toStdString();
+      return EXIT_FAILURE;
+    }
+
+    if ( myCTK.schemaVersionLoaded() != QString("") )
+    {
+      std::cerr << "Schema tag should be empty in old schema\n";
+      std::cerr << "Instead we got: (" << myCTK.schemaVersionLoaded().toStdString() << ")\n";
+      return EXIT_FAILURE;
+    }
+
+    myCTK.closeDatabase();
+  }
+  catch (std::exception e)
+    {
+    std::cerr << "Error when opening the data base file: " << databaseFileName.toStdString()
+        << " error: " << e.what();
+    return EXIT_FAILURE;
+    }
+
+  // now try opening it and updating the schema 
+  try
+  {
+    ctkDICOMDatabase myCTK( databaseFileName );
+
+    if ( myCTK.schemaVersionLoaded() == myCTK.schemaVersion() )
+    {
+      std::cerr << "Schema version should Not match\n";
+      return EXIT_FAILURE;
+    }
+
+    if ( !myCTK.updateSchema() )
+    {
+      std::cerr << "Could not update schema\n";
+      return EXIT_FAILURE;
+    }
+
+    if ( myCTK.schemaVersionLoaded() != myCTK.schemaVersion() )
+    {
+      std::cerr << "Schema version should match\n";
+      return EXIT_FAILURE;
+    }
+
+    myCTK.closeDatabase();
+  }
+  catch (std::exception e)
+    {
+    std::cerr << "Error when re-opening the data base file: " << databaseFileName.toStdString()
+        << " error: " << e.what();
+    return EXIT_FAILURE;
+    }
+
+
+  return EXIT_SUCCESS;
+}

Libs/DICOM/Core/ctkDICOMDatabase.cpp

@@ -118,6 +118,9 @@ public:
   QString LastSeriesInstanceUID;
   int LastPatientUID;
 
+  /// resets the variables to new inserts won't be fooled by leftover values
+  void resetLastInsertedValues();
+
   /// parallel inserts are not allowed (yet)
   QMutex insertMutex;
 
@@ -142,8 +145,19 @@ ctkDICOMDatabasePrivate::ctkDICOMDatabasePrivate(ctkDICOMDatabase& o): q_ptr(&o)
 {
   this->thumbnailGenerator = NULL;
   this->LoggedExecVerbose = false;
-  this->LastPatientUID = -1;
   this->TagCacheVerified = false;
+  this->resetLastInsertedValues();
+}
+
+//------------------------------------------------------------------------------
+void ctkDICOMDatabasePrivate::resetLastInsertedValues()
+{
+  this->LastPatientID = QString("");
+  this->LastPatientsName = QString("");
+  this->LastPatientsBirthDate = QString("");
+  this->LastStudyInstanceUID = QString("");
+  this->LastSeriesInstanceUID = QString("");
+  this->LastPatientUID = -1;
 }
 
 //------------------------------------------------------------------------------
@@ -246,6 +260,9 @@ void ctkDICOMDatabase::openDatabase(const QString databaseFile, const QString& c
           return;
         }
     }
+  d->resetLastInsertedValues();
+
+
   if (!isInMemory())
     {
       QFileSystemWatcher* watcher = new QFileSystemWatcher(QStringList(databaseFile),this);
@@ -382,10 +399,47 @@ QStringList ctkDICOMDatabasePrivate::filenames(QString table)
 bool ctkDICOMDatabase::initializeDatabase(const char* sqlFileName)
 {
   Q_D(ctkDICOMDatabase);
+
+  d->resetLastInsertedValues();
+
+  // remove any existing schema info - this handles the case where an
+  // old schema should be loaded for testing.
+  QSqlQuery dropSchemaInfo(d->Database);
+  d->loggedExec( dropSchemaInfo, QString("DROP TABLE IF EXISTS 'SchemaInfo';") );
   return d->executeScript(sqlFileName);
 }
 
 //------------------------------------------------------------------------------
+QString ctkDICOMDatabase::schemaVersionLoaded()
+{
+  Q_D(ctkDICOMDatabase);
+  /// look for the version info in the database
+  QSqlQuery versionQuery(d->Database);
+  if ( !d->loggedExec( versionQuery, QString("SELECT Version from SchemaInfo;") ) )
+    {
+    return QString("");
+    }
+
+  if (versionQuery.next())
+    {
+    return versionQuery.value(0).toString();
+    }
+
+  return QString("");
+}
+
+
+//------------------------------------------------------------------------------
+bool ctkDICOMDatabase::updateSchemaIfNeeded(const char* schemaFile)
+{
+  if ( schemaVersionLoaded() != schemaVersion() )
+    {
+    return this->updateSchema(schemaFile);
+    }
+  return false;
+}
+
+//------------------------------------------------------------------------------
 bool ctkDICOMDatabase::updateSchema(const char* schemaFile)
 {
   // backup filelist
@@ -395,16 +449,26 @@ bool ctkDICOMDatabase::updateSchema(const char* schemaFile)
   Q_D(ctkDICOMDatabase);
   d->createBackupFileList();
  
+  d->resetLastInsertedValues();
   this->initializeDatabase(schemaFile);
 
   QStringList allFiles = d->filenames("Filenames_backup");
+  emit schemaUpdateStarted(allFiles.length());
+  
+  int progressValue = 0;
   foreach(QString file, allFiles)
   {
+    emit schemaUpdateProgress(progressValue);
+    emit schemaUpdateProgress(file);
+
     // TODO: use QFuture
     this->insert(file,false,false,true);
+
+    progressValue++;
   }
   // TODO: check better that everything is ok
   d->removeBackupFileList();
+  emit schemaUpdated();
   return true;
 
 }
@@ -1203,7 +1267,7 @@ bool ctkDICOMDatabase::removeSeries(const QString& seriesInstanceUID)
 
   this->cleanup();
 
-  d->LastSeriesInstanceUID = "";
+  d->resetLastInsertedValues();
 
   return true;
 }
@@ -1242,7 +1306,7 @@ bool ctkDICOMDatabase::removeStudy(const QString& studyInstanceUID)
           result = false;
         }
     }
-  d->LastStudyInstanceUID = "";
+  d->resetLastInsertedValues();
   return result;
 }
 
@@ -1269,10 +1333,7 @@ bool ctkDICOMDatabase::removePatient(const QString& patientID)
           result = false;
         }
     }
-  d->LastPatientID = "";
-  d->LastPatientsName = "";
-  d->LastPatientsBirthDate = "";
-  d->LastPatientUID = -1;
+  d->resetLastInsertedValues();
   return result;
 }
 

Libs/DICOM/Core/ctkDICOMDatabase.h

@@ -100,8 +100,9 @@ public:
   ///        written to disk at all but instead only kept in memory (and
   ///        thus expires after destruction of this object).
   /// @param connectionName The database connection name.
+  /// @param update the schema if it is found to be out of date
   Q_INVOKABLE virtual void openDatabase(const QString databaseFile,
-                                        const QString& connectionName = "DICOM-DB" );
+                                        const QString& connectionName = "DICOM-DB");
 
   ///
   /// close the database. It must not be used afterwards.
@@ -113,6 +114,17 @@ public:
   /// updates the database schema and reinserts all existing files
   Q_INVOKABLE bool updateSchema(const char* schemaFile = ":/dicom/dicom-schema.sql");
 
+  /// updates the database schema only if the versions don't match
+  /// Returns true if schema was updated
+  Q_INVOKABLE bool updateSchemaIfNeeded(const char* schemaFile = ":/dicom/dicom-schema.sql");
+
+  /// returns the schema version needed by the current version of this code
+  Q_INVOKABLE QString schemaVersion() { return QString("0.5"); };
+
+  /// returns the schema version for the currently open database
+  /// in order to support schema updating
+  Q_INVOKABLE QString schemaVersionLoaded();
+
   ///
   /// \brief database accessors
   Q_INVOKABLE QStringList patients ();
@@ -201,6 +213,13 @@ public:
 
 Q_SIGNALS:
   void databaseChanged();
+  /// Indicates that the schema is about to be updated and how many files will be processed
+  void schemaUpdateStarted(int);
+  /// Indicates progress in updating schema (int is file number, string is file name)
+  void schemaUpdateProgress(int);
+  void schemaUpdateProgress(QString);
+  /// Indicates schema update finished
+  void schemaUpdated();
 
 protected:
   QScopedPointer<ctkDICOMDatabasePrivate> d_ptr;

Libs/DICOM/Widgets/ctkDICOMAppWidget.cpp

@@ -193,6 +193,54 @@ ctkDICOMAppWidget::~ctkDICOMAppWidget()
 }
 
 //----------------------------------------------------------------------------
+void ctkDICOMAppWidget::updateDatabaseSchemaIfNeeded()
+{
+
+  Q_D(ctkDICOMAppWidget);  
+
+  if ( d->DICOMDatabase->schemaVersion() != d->DICOMDatabase->schemaVersionLoaded() )
+    {
+    QProgressDialog* progress = new QProgressDialog("DICOM Schema Update", "Cancel", 0, 100, this,
+                           Qt::WindowTitleHint | Qt::WindowSystemMenuHint);
+    // We don't want the progress dialog to resize itself, so we bypass the label
+    // by creating our own
+    QLabel* progressLabel = new QLabel(tr("Initialization..."));
+    progress->setLabel(progressLabel);
+#ifdef Q_WS_MAC
+    // BUG: avoid deadlock of dialogs on mac
+    progress->setWindowModality(Qt::NonModal);
+#else
+    progress->setWindowModality(Qt::ApplicationModal);
+#endif
+    progress->setMinimumDuration(0);
+    progress->setValue(0);
+    progress->show();
+
+    // TODO - cancel?
+    //connect(progress, SIGNAL(canceled()), d->DICOMIndexer.data(), SLOT(cancel()));
+
+    connect(d->DICOMDatabase.data(), SIGNAL(schemaUpdateStarted(int)),
+            progress, SLOT(setMaximum(int)));
+    connect(d->DICOMDatabase.data(), SIGNAL(schemaUpdateProgress(int)),
+            progress, SLOT(setValue(int)));
+    connect(d->DICOMDatabase.data(), SIGNAL(schemaUpdateProgress(QString)),
+            progressLabel, SLOT(setText(QString)));
+    connect(d->DICOMDatabase.data(), SIGNAL(progress(int)),
+            this, SLOT(onProgress(int)));
+
+    // close the dialog
+    connect(d->DICOMDatabase.data(), SIGNAL(schemaUpdated()),
+            progress, SLOT(close()));
+    // reset the database to show new data
+    connect(d->DICOMDatabase.data(), SIGNAL(schemaUpdated()),
+            &d->DICOMModel, SLOT(reset()));
+
+    d->DICOMDatabase->updateSchema();
+    }
+
+}
+
+//----------------------------------------------------------------------------
 void ctkDICOMAppWidget::setDatabaseDirectory(const QString& directory)
 {
   Q_D(ctkDICOMAppWidget);  
@@ -216,6 +264,9 @@ void ctkDICOMAppWidget::setDatabaseDirectory(const QString& directory)
     d->DICOMDatabase->closeDatabase();
     return;
     }
+
+  // update the database schema if needed and provide progress
+  this->updateDatabaseSchemaIfNeeded();
   
   d->DICOMModel.setDatabase(d->DICOMDatabase->database());
   d->DICOMModel.setEndLevel(ctkDICOMModel::SeriesType);

Libs/DICOM/Widgets/ctkDICOMAppWidget.h

@@ -44,6 +44,11 @@ public:
 
   QString databaseDirectory() const;
 
+  /// Updates schema of loaded database to match the one
+  /// coded by the current version of ctkDICOMDatabase.
+  /// Also provides a dialog box for progress
+  void updateDatabaseSchemaIfNeeded();
+
   /// Setting search widget pop-up mode
   /// Default value is false. Setting it to true will make
   /// search widget to be displayed as pop-up widget