Implemented callback timeout for Windows and Linux, tested under Linux.

git-svn-id: https://svn.microneil.com/svn/CodeDweller/branches/adeniz_1@42 d34b734f-a00e-4b39-a726-e4eeb87269ab

service.cpp

@@ -142,7 +142,10 @@ namespace CodeDweller {
 
   std::vector<Service::Callback *> Service::stopCallbacks;
 
-  Service::Service() {
+  int Service::callbackTimeout_ms = 30 * 1000;
+
+  Service::Service() :
+    callbacksActive(true) {
   }
 
   Service &Service::getInstance() {
@@ -381,6 +384,10 @@ namespace CodeDweller {
     return cmdLineArgs;
   }
 
+  void Service::setCallbackTimeout_ms(int timeout_ms) {
+    callbackTimeout_ms = (timeout_ms < 1 ? 1 : timeout_ms);
+  }
+
   void Service::onPauseCall(Callback &pauseFunctor) {
     std::lock_guard<std::mutex> scopeMutex(objectMutex);
     pauseCallbacks.push_back(&pauseFunctor);
@@ -445,16 +452,19 @@ namespace CodeDweller {
       serviceStatus.dwCurrentState = SERVICE_PAUSE_PENDING;
       serviceStatus.dwWin32ExitCode = NO_ERROR;
       serviceStatus.dwCheckPoint = 1;
-      serviceStatus.dwWaitHint = 5000;
+      serviceStatus.dwWaitHint = callbackTimeout_ms;
       (void) SetServiceStatus(serviceStatusHandle, &serviceStatus);
 
       pauseReceived = true;
 
       for (auto callback : pauseCallbacks) {
 	(*callback)();
+	serviceStatus.dwCheckPoint++;
+	(void) SetServiceStatus(serviceStatusHandle, &serviceStatus);
       }
 
       serviceStatus.dwControlsAccepted =
+	SERVICE_ACCEPT_SHUTDOWN |
 	SERVICE_ACCEPT_STOP |
 	SERVICE_ACCEPT_PAUSE_CONTINUE;
       serviceStatus.dwCurrentState = SERVICE_PAUSED;
@@ -476,16 +486,19 @@ namespace CodeDweller {
       serviceStatus.dwCurrentState = SERVICE_CONTINUE_PENDING;
       serviceStatus.dwWin32ExitCode = NO_ERROR;
       serviceStatus.dwCheckPoint = 1;
-      serviceStatus.dwWaitHint = 5000;
+      serviceStatus.dwWaitHint = callbackTimeout_ms;
       (void) SetServiceStatus(serviceStatusHandle, &serviceStatus);
 
       resumeReceived = true;
 
       for (auto callback : resumeCallbacks) {
 	(*callback)();
+	serviceStatus.dwCheckPoint++;
+	(void) SetServiceStatus(serviceStatusHandle, &serviceStatus);
       }
 
       serviceStatus.dwControlsAccepted =
+	SERVICE_ACCEPT_SHUTDOWN |
 	SERVICE_ACCEPT_STOP |
 	SERVICE_ACCEPT_PAUSE_CONTINUE;
       serviceStatus.dwCurrentState = SERVICE_RUNNING;
@@ -496,10 +509,13 @@ namespace CodeDweller {
       break;
 
     case SERVICE_CONTROL_STOP:
+    case SERVICE_CONTROL_SHUTDOWN:
 
 #ifdef DEBUG_LOG_FILE
       logStream.open(DEBUG_LOG_FILE, std::fstream::app);
-      logStream << "processCtrlMessage.  Received STOP" << std::endl;
+      logStream << "processCtrlMessage.  Received "
+		<< (message == SERVICE_CONTROL_STOP ? "STOP" : "SHUTDOWN")
+		<< std::endl;
       logStream.close();
 #endif
 
@@ -507,13 +523,15 @@ namespace CodeDweller {
       serviceStatus.dwCurrentState = SERVICE_STOP_PENDING;
       serviceStatus.dwWin32ExitCode = NO_ERROR;
       serviceStatus.dwCheckPoint = 1;
-      serviceStatus.dwWaitHint = 5000;
+      serviceStatus.dwWaitHint = callbackTimeout_ms;
       (void) SetServiceStatus(serviceStatusHandle, &serviceStatus);
 
       stopReceived = true;
 
       for (auto callback : stopCallbacks) {
 	(*callback)();
+	serviceStatus.dwCheckPoint++;
+	(void) SetServiceStatus(serviceStatusHandle, &serviceStatus);
       }
 
       break;
@@ -545,28 +563,119 @@ namespace CodeDweller {
       switch (sigNum) {
 
       case SIGTSTP:
-	pauseReceived = true;
+	{
 
-	for (auto callback : pauseCallbacks) {
-	  (*callback)();
-	}
+	  pauseReceived = true;
+	  callbacksActive = true;
+
+	  // Get the timeout time.
+	  timeoutTime =
+	    std::chrono::steady_clock::now() +
+	    std::chrono::milliseconds(callbackTimeout_ms);
+
+	  // Start watchdog thread.
+	  std::thread watchdogThread(&Service::watchdog, this);
+
+#ifdef DEBUG_LOG_FILE
+	  std::ofstream logStream;
+	  auto startTime = std::chrono::steady_clock::now();
 
+	  logStream.open(DEBUG_LOG_FILE, std::fstream::app);
+	  logStream << "Service::processMessages.  Calling Pause callbacks--"
+		    << std::endl;
+#endif
+	  for (auto callback : pauseCallbacks) {
+	    (*callback)();
+	    timeoutTime =
+	      std::chrono::steady_clock::now() +
+	      std::chrono::milliseconds(callbackTimeout_ms);
+#ifdef DEBUG_LOG_FILE
+	    logStream << "Service::processMessages.  "
+		      << "Cumulative time after callback (ms):  "
+		      << std::chrono::duration_cast<std::chrono::milliseconds>
+	      (std::chrono::steady_clock::now() - startTime).count()
+		      << std::endl;
+#endif
+	  }
+
+	  callbacksActive = false;
+
+#ifdef DEBUG_LOG_FILE
+	    logStream << "Service::processMessages.  "
+		      << "Cumulative time after all callback (ms):  "
+		      << std::chrono::duration_cast<std::chrono::milliseconds>
+	      (std::chrono::steady_clock::now() - startTime).count()
+		      << std::endl;
+#endif
+	  watchdogThread.join();
+
+#ifdef DEBUG_LOG_FILE
+	    logStream << "Service::processMessages.  "
+		      <<" Cumulative time after joining watchdog (ms):  "
+		      << std::chrono::duration_cast<std::chrono::milliseconds>
+	      (std::chrono::steady_clock::now() - startTime).count()
+		      << std::endl;
+	  logStream.close();
+#endif
+
+	}
+	
 	break;
 
       case SIGCONT:
-	resumeReceived = true;
+	{
+
+	  resumeReceived = true;
+	  callbacksActive = true;
+
+	  // Get the timeout time.
+	  timeoutTime =
+	    std::chrono::steady_clock::now() +
+	    std::chrono::milliseconds(callbackTimeout_ms);
+
+	  // Start watchdog thread.
+	  std::thread watchdogThread(&Service::watchdog, this);
+
+	  for (auto callback : resumeCallbacks) {
+	    (*callback)();
+	    timeoutTime =
+	      std::chrono::steady_clock::now() +
+	      std::chrono::milliseconds(callbackTimeout_ms);
+	  }
+
+	  callbacksActive = false;
+
+	  watchdogThread.join();
 
-	for (auto callback : resumeCallbacks) {
-	  (*callback)();
 	}
 
 	break;
 
       case SIGTERM:
-	stopReceived = true;
+	{
+
+	  stopReceived = true;
+	  callbacksActive = true;
+
+	  // Get the timeout time.
+	  timeoutTime =
+	    std::chrono::steady_clock::now() +
+	    std::chrono::milliseconds(callbackTimeout_ms);
+
+	  // Start watchdog thread.
+	  std::thread watchdogThread(&Service::watchdog, this);
+
+	  for (auto callback : stopCallbacks) {
+	    (*callback)();
+	    timeoutTime =
+	      std::chrono::steady_clock::now() +
+	      std::chrono::milliseconds(callbackTimeout_ms);
+	  }
+
+	  callbacksActive = false;
+
+	  watchdogThread.join();
 
-	for (auto callback : stopCallbacks) {
-	  (*callback)();
 	}
 
 	// Exit.
@@ -583,5 +692,56 @@ namespace CodeDweller {
 
   }
 
+  void Service::watchdog() {
+
+#ifdef DEBUG_LOG_FILE
+    auto startTime = std::chrono::steady_clock::now();
+    std::ofstream logStream;
+      logStream.open(DEBUG_LOG_FILE, std::fstream::app);
+      logStream << "**Service::watchdog.  Thread starting." << std::endl;
+      logStream.close();
+#endif
+
+    // Sleep and check whether the process should exit.
+    std::chrono::milliseconds sleepTime(100);
+
+    while (std::chrono::steady_clock::now() < timeoutTime) {
+      std::this_thread::sleep_for(sleepTime);
+      if (!callbacksActive) {
+#ifdef DEBUG_LOG_FILE
+	logStream.open(DEBUG_LOG_FILE, std::fstream::app);
+	logStream << "**Service::watchdog.  Exiting at  "
+		  << std::chrono::duration_cast<std::chrono::milliseconds>
+	  (std::chrono::steady_clock::now() - startTime).count()
+		  << " ms after starting"
+		  << std::endl;
+	logStream.close();
+#endif
+	return;
+      }
+#ifdef DEBUG_LOG_FILE
+      logStream.open(DEBUG_LOG_FILE, std::fstream::app);
+      logStream << "Service::watchdog.  Cumulative time since starting (ms):  "
+		<< std::chrono::duration_cast<std::chrono::milliseconds>
+	(std::chrono::steady_clock::now() - startTime).count()
+		<< std::endl;
+      logStream.close();
+#endif
+    }
+
+    // Timeout expired.
+#ifdef DEBUG_LOG_FILE
+    logStream.open(DEBUG_LOG_FILE, std::fstream::app);
+    logStream << "**Service::watchdog.  Callback timeout expired.  "
+	      << "Cumulative time (ms):  "
+	      << std::chrono::duration_cast<std::chrono::milliseconds>
+      (std::chrono::steady_clock::now() - startTime).count()
+	      << std::endl;
+    logStream.close();
+#endif
+    std::exit(EXIT_FAILURE);
+
+  }
+
 #endif
 }

service.hpp

@@ -30,7 +30,13 @@
 #define SERVICE_HPP
 
 #ifdef WIN32
+
 #include <windows.h>
+
+#else
+
+#include <chrono>
+
 #endif
 
 #include <string>
@@ -85,8 +91,8 @@ namespace CodeDweller {
 	    the Resume message has no effect.</li>
 
 	    <li>Stop.  This is the posix TERM signal or Windows Stop
-	    message.  This instructs the service to shut down and
-	    exit.</li>
+	    or Shutdown message.  This instructs the service to shut
+	    down and exit.</li>
 
 	  </ol>
 
@@ -169,6 +175,16 @@ namespace CodeDweller {
     //
     static const std::vector<std::string> &arguments();
 
+    /// Set the timeout for executing one callback.
+    //
+    // If a callback takes longer than the specified timeout, the
+    // service aborts.
+    //
+    // \param[in] timeout_ms is the timeout in milliseconds.  If a
+    // value less than 1 is specified, a value of 1 is used.
+    //
+    static void setCallbackTimeout_ms(int timeout_ms);
+
   private:
 
     /// Private constructor prevents instantiation.
@@ -241,8 +257,16 @@ namespace CodeDweller {
     void processCtrlMessage(DWORD message);
 
 #else
+
     /// Thread start function to receive and process messages.
     void processMessages();
+
+    /// Thread start function for the watchdog thread.
+    //
+    // This thread sleeps until timeoutTime, and then causes the
+    // process to exit.
+    void watchdog();
+
 #endif
 
     /// Command-line arguments.
@@ -266,6 +290,9 @@ namespace CodeDweller {
     /// Functions to invoke when the Stop is received.
     static std::vector<Callback *> stopCallbacks;
 
+    /// Callback timeout.
+    static int callbackTimeout_ms;
+
 #ifdef WIN32
 
     /// Status of the service.
@@ -287,6 +314,12 @@ namespace CodeDweller {
 
     friend int ::main(int argc, char *argv[]);
 
+    /// Absolute time at which the callback timeout expires.
+    std::chrono::time_point<std::chrono::steady_clock> timeoutTime;
+
+    /// True if the callbacks are being invoked.
+    bool callbacksActive;
+
 #endif
 
   };