瀏覽代碼

ENH: Improve DICOM import speed

Testing DICOM import speed on Windows with Qt4, it turned out that excessive logging and progress updates slowed down the import
by almost an order of magnitude.

Speed comparison example:

Baseline (without any changes):
"DICOM indexer has successfully processed 801 files [72.72s]"

After changing ctkDICOMIndexer to update progress report at every 10% increase (not at every processed file):
"DICOM indexer has successfully processed 801 files [37.53s]"

After cutting down on debug logs:
"DICOM indexer has successfully processed 801 files [11.46s]"
Andras Lasso 8 年之前
父節點
當前提交
29fd54f73f
共有 3 個文件被更改,包括 57 次插入18 次删除
  1. 49 16
      Libs/DICOM/Core/ctkDICOMDatabase.cpp
  2. 8 1
      Libs/DICOM/Core/ctkDICOMIndexer.cpp
  3. 0 1
      Libs/DICOM/Widgets/ctkDICOMBrowser.cpp

+ 49 - 16
Libs/DICOM/Core/ctkDICOMDatabase.cpp

@@ -1065,7 +1065,10 @@ void ctkDICOMDatabase::insert ( const QString& filePath, bool storeFile, bool ge
       return;
       return;
     }
     }
 
 
-  logger.debug( "Processing " + filePath );
+  if (d->LoggedExecVerbose)
+    {
+    logger.debug( "Processing " + filePath );
+    }
 
 
   ctkDICOMItem ctkDataset;
   ctkDICOMItem ctkDataset;
 
 
@@ -1188,7 +1191,10 @@ void ctkDICOMDatabasePrivate::insertSeries(const ctkDICOMItem& ctkDataset, QStri
   QSqlQuery checkSeriesExistsQuery (Database);
   QSqlQuery checkSeriesExistsQuery (Database);
   checkSeriesExistsQuery.prepare ( "SELECT * FROM Series WHERE SeriesInstanceUID = ?" );
   checkSeriesExistsQuery.prepare ( "SELECT * FROM Series WHERE SeriesInstanceUID = ?" );
   checkSeriesExistsQuery.bindValue ( 0, seriesInstanceUID );
   checkSeriesExistsQuery.bindValue ( 0, seriesInstanceUID );
-  logger.warn ( "Statement: " + checkSeriesExistsQuery.lastQuery() );
+  if (this->LoggedExecVerbose)
+    {
+    logger.warn ( "Statement: " + checkSeriesExistsQuery.lastQuery() );
+    }
   checkSeriesExistsQuery.exec();
   checkSeriesExistsQuery.exec();
   if(!checkSeriesExistsQuery.next())
   if(!checkSeriesExistsQuery.next())
     {
     {
@@ -1342,10 +1348,16 @@ void ctkDICOMDatabasePrivate::insert( const ctkDICOMItem& ctkDataset, const QStr
       return;
       return;
     }
     }
   bool found = fileExistsQuery.next();
   bool found = fileExistsQuery.next();
-  qDebug() << "inserting filePath: " << filePath;
+  if (this->LoggedExecVerbose)
+    {
+    qDebug() << "inserting filePath: " << filePath;
+    }
   if (!found)
   if (!found)
     {
     {
+    if (this->LoggedExecVerbose)
+      {
       qDebug() << "database filename for " << sopInstanceUID << " is empty - we should insert on top of it";
       qDebug() << "database filename for " << sopInstanceUID << " is empty - we should insert on top of it";
+      }
     }
     }
   else
   else
     {
     {
@@ -1415,7 +1427,10 @@ void ctkDICOMDatabasePrivate::insert( const ctkDICOMItem& ctkDataset, const QStr
 
 
       if(filePath.isEmpty())
       if(filePath.isEmpty())
         {
         {
-          logger.debug ( "Saving file: " + filename );
+          if (this->LoggedExecVerbose)
+            {
+            logger.debug ( "Saving file: " + filename );
+            }
 
 
           if ( !ctkDataset.SaveToFile( filename) )
           if ( !ctkDataset.SaveToFile( filename) )
             {
             {
@@ -1429,8 +1444,10 @@ void ctkDICOMDatabasePrivate::insert( const ctkDICOMItem& ctkDataset, const QStr
 
 
           QFile currentFile( filePath );
           QFile currentFile( filePath );
           currentFile.copy(filename);
           currentFile.copy(filename);
-          logger.debug( "Copy file from: " + filePath );
-          logger.debug( "Copy file to  : " + filename );
+          if (this->LoggedExecVerbose)
+            {
+            logger.debug("Copy file from: " + filePath + " to: " + filename);
+            }
         }
         }
     }
     }
 
 
@@ -1448,7 +1465,10 @@ void ctkDICOMDatabasePrivate::insert( const ctkDICOMItem& ctkDataset, const QStr
            || LastPatientsBirthDate != patientsBirthDate
            || LastPatientsBirthDate != patientsBirthDate
            || LastPatientsName != patientsName )
            || LastPatientsName != patientsName )
         {
         {
-          qDebug() << "This looks like a different patient from last insert: " << patientID;
+          if (this->LoggedExecVerbose)
+            {
+            qDebug() << "This looks like a different patient from last insert: " << patientID;
+            }
           // Ok, something is different from last insert, let's insert him if he's not
           // Ok, something is different from last insert, let's insert him if he's not
           // already in the db.
           // already in the db.
 
 
@@ -1464,7 +1484,10 @@ void ctkDICOMDatabasePrivate::insert( const ctkDICOMItem& ctkDataset, const QStr
           LastPatientsName = patientsName;
           LastPatientsName = patientsName;
         }
         }
 
 
-      qDebug() << "Going to insert this instance with dbPatientID: " << dbPatientID;
+      if (this->LoggedExecVerbose)
+        {
+        qDebug() << "Going to insert this instance with dbPatientID: " << dbPatientID;
+        }
 
 
       // Patient is in now. Let's continue with the study
       // Patient is in now. Let's continue with the study
 
 
@@ -1494,7 +1517,10 @@ void ctkDICOMDatabasePrivate::insert( const ctkDICOMItem& ctkDataset, const QStr
           checkImageExistsQuery.prepare ( "SELECT * FROM Images WHERE Filename = ?" );
           checkImageExistsQuery.prepare ( "SELECT * FROM Images WHERE Filename = ?" );
           checkImageExistsQuery.bindValue ( 0, filename );
           checkImageExistsQuery.bindValue ( 0, filename );
           checkImageExistsQuery.exec();
           checkImageExistsQuery.exec();
-          qDebug() << "Maybe add Instance";
+          if (this->LoggedExecVerbose)
+            {
+            qDebug() << "Maybe add Instance";
+            }
           if(!checkImageExistsQuery.next())
           if(!checkImageExistsQuery.next())
             {
             {
               QSqlQuery insertImageStatement ( Database );
               QSqlQuery insertImageStatement ( Database );
@@ -1510,7 +1536,10 @@ void ctkDICOMDatabasePrivate::insert( const ctkDICOMItem& ctkDataset, const QStr
 
 
               // let users of this class track when things happen
               // let users of this class track when things happen
               emit q->instanceAdded(sopInstanceUID);
               emit q->instanceAdded(sopInstanceUID);
-              qDebug() << "Instance Added";
+              if (this->LoggedExecVerbose)
+                {
+                qDebug() << "Instance Added";
+                }
             }
             }
         }
         }
 
 
@@ -1631,20 +1660,24 @@ bool ctkDICOMDatabase::removeSeries(const QString& seriesInstanceUID)
             }
             }
           if (QFile( dbFilePath ).remove())
           if (QFile( dbFilePath ).remove())
             {
             {
-              logger.debug("Removed file " + dbFilePath );
+              if (d->LoggedExecVerbose)
+                {
+                logger.debug("Removed file " + dbFilePath );
+                }
             }
             }
           else
           else
             {
             {
               logger.warn("Failed to remove file " + dbFilePath );
               logger.warn("Failed to remove file " + dbFilePath );
             }
             }
         }
         }
-      if (QFile( thumbnailToRemove ).remove())
-        {
-          logger.debug("Removed thumbnail " + thumbnailToRemove);
-        }
-      else
+      // Remove thumbnail (if exists)
+      QFile thumbnailFile(thumbnailToRemove);
+      if (thumbnailFile.exists())
         {
         {
+        if (!thumbnailFile.remove())
+          {
           logger.warn("Failed to remove thumbnail " + thumbnailToRemove);
           logger.warn("Failed to remove thumbnail " + thumbnailToRemove);
+          }
         }
         }
     }
     }
 
 

+ 8 - 1
Libs/DICOM/Core/ctkDICOMIndexer.cpp

@@ -134,10 +134,17 @@ void ctkDICOMIndexer::addListOfFiles(ctkDICOMDatabase& ctkDICOMDatabase,
   timeProbe.start();
   timeProbe.start();
   d->Canceled = false;
   d->Canceled = false;
   int CurrentFileIndex = 0;
   int CurrentFileIndex = 0;
+  int lastReportedPercent = 0;
   foreach(QString filePath, listOfFiles)
   foreach(QString filePath, listOfFiles)
   {
   {
     int percent = ( 100 * CurrentFileIndex ) / listOfFiles.size();
     int percent = ( 100 * CurrentFileIndex ) / listOfFiles.size();
-    emit this->progress(percent);
+    if (lastReportedPercent / 10 < percent / 10)
+      {
+      // Reporting progress has a huge overhead (pending events are processed,
+      // database is updated), therefore only report progress at every 10% increase
+      emit this->progress(percent);
+      lastReportedPercent = percent;
+      }
     this->addFile(ctkDICOMDatabase, filePath, destinationDirectoryName);
     this->addFile(ctkDICOMDatabase, filePath, destinationDirectoryName);
     CurrentFileIndex++;
     CurrentFileIndex++;
 
 

+ 0 - 1
Libs/DICOM/Widgets/ctkDICOMBrowser.cpp

@@ -423,7 +423,6 @@ ctkDICOMTableManager* ctkDICOMBrowser::dicomTableManager()
 //----------------------------------------------------------------------------
 //----------------------------------------------------------------------------
 void ctkDICOMBrowser::onFileIndexed(const QString& filePath)
 void ctkDICOMBrowser::onFileIndexed(const QString& filePath)
 {
 {
-  qDebug() << "Indexing \n\n\n\n" << filePath <<"\n\n\n";
 }
 }
 
 
 //----------------------------------------------------------------------------
 //----------------------------------------------------------------------------