SNFMulti/SNFMulti.cpp

// SNFMulti.cpp
//
// (C) Copyright 2006 - 2020 ARM Research Labs, LLC
// See www.armresearch.com for the copyright terms.
//
// 20060121_M
//
// See SNFMulti.hpp for history and detailed notes.

#include <sys/types.h>
#include <sys/stat.h>
#include <ctime>
#include <cstring>
#include <cstdlib>

#include <sstream>
#include "SNFMulti.hpp"
#include "snf_saccades.hpp"

#include "../CodeDweller/timing.hpp"

//#include "../nvwa-0.6/nvwa/debug_new.h"

namespace cd = codedweller;

//// Version Info

const char* SNF_ENGINE_VERSION = "SNFMulti Engine Version 3.2.1 Build: " __DATE__ " " __TIME__;

//// Script Caller Methods

const cd::ThreadType ScriptCaller::Type("Script Caller");                       // Script caller thread type mnemonic.

const cd::ThreadState ScriptCaller::CallingSystem("In system()");               // Script caller "CallingSystem" state.
const cd::ThreadState ScriptCaller::PendingGuardTime("Guard Time");             // Script caller "GuardTime" state.
const cd::ThreadState ScriptCaller::StandingBy("Standby");                      // Script caller "Standby" state.
const cd::ThreadState ScriptCaller::Disabled("Disabled");                       // State when unable to run.


const int ScriptGuardDefault = 180000;                                          // 3 Minute Default Guard Time.

ScriptCaller::ScriptCaller(std::string S) :                                     // Script caller constructor (with name).
  Thread(ScriptCaller::Type, S),                                                // Set up the thread type and name.
  GuardTimer(ScriptGuardDefault),                                               // Initialize the guard time.
  GoFlag(false),                                                                // Not ready to go yet.
  DieFlag(false),                                                               // Not ready to die yet.
  myLastResult(0) {                                                             // No last result yet.
    run();                                                                      // Launch the thread.
}

ScriptCaller::~ScriptCaller() {                                                 // Destructor.
    DieFlag = true;                                                             // Set the die flag.
    cd::Sleeper WaitATic(1000);                                                 // One second sleeper.
    for(int x = 10; x > 0; x--) {                                               // We don't join, we might get stuck.
        if(false == isRunning()) break;                                         // If we're still running then wait
        WaitATic();                                                             // up to 10 seconds, then just exit.
    }                                                                           // If the thread is stuck it will
}                                                                               // just get closed.

std::string ScriptCaller::ScriptToRun() {                                       // Safely grab the SystemCallText.
    cd::ScopeMutex Freeze(MyMutex);                                             // Protect the string.
    return SystemCallText;                                                      // Grab a copy of the text.
}

bool ScriptCaller::hasGuardExpired() {                                          // True if guard time has expired.
    cd::ScopeMutex Freeze(MyMutex);                                             // Protect the timer.
    return GuardTimer.isExpired();                                              // If it has expired we're true.
}

void ScriptCaller::SystemCall(std::string S) {                                  // Set the SystemCall text.
    cd::ScopeMutex Freeze(MyMutex);                                             // Protect the string object.
    SystemCallText = S;                                                         // Set it's data.
}

const int MinimumGuardTime = 60000;                                             // Minimum Guard Time 1 minute.
void ScriptCaller::GuardTime(int T) {                                           // Set the Guard Time.
    if(MinimumGuardTime > T) T = MinimumGuardTime;                              // Enforce our lower limit.
    cd::ScopeMutex Freeze(MyMutex);                                             // Protect the Guard Timer.
    GuardTimer.setDuration(T);                                                  // Set the duration.
    GuardTimer.restart();                                                       // Restart the timer.
}

void ScriptCaller::trigger() {                                                  // Trigger the system() call.
    GoFlag = true;                                                              // Set the flag.
}

int ScriptCaller::LastResult() {                                                // Return the result code from
    return myLastResult;                                                        // the last system() call.
}

void ScriptCaller::myTask() {                                                   // Safely call system() when triggered.
    cd::Sleeper WaitATic(1000);                                                 // One second sleeper.
    while(false == DieFlag) {                                                   // While it's not time to die:
        WaitATic();                                                             // Pause for 1 sec each round.
        std::string ScriptThisRound = ScriptToRun();                            // Grab the current script.
        if(0 < ScriptToRun().length()) {                                        // If script text is defined and
            if(true == GoFlag) {                                                // If GoFlag is triggered and
                if(hasGuardExpired()) {                                         // Guard time is expired:
                    CurrentThreadState(CallingSystem);                          // Publish our state.
                    myLastResult = system(ScriptThisRound.c_str());             // Make the system call.
                    GoFlag = false;                                             // Done with that trigger.
                    GuardTimer.restart();                                       // Restart our Guard Time.
                } else {                                                        // If we're waiting for Guard Time:
                    CurrentThreadState(PendingGuardTime);                       // publish that state and hold down
                    GoFlag = false;                                             // the trigger signal (no stale go).
                }
            } else {                                                            // If nothing is triggered yet then
                CurrentThreadState(StandingBy);                                 // we are standing by.
            }
        } else {                                                                // If we have no script to run then
            CurrentThreadState(Disabled);                                       // we are disabled.
        }
    }
}

//// Rulebase Reloader Methods

// How to get timestamps on critical files.

time_t getFileTimestamp(std::string FileName) {
    struct stat FileNameStat;                                                   // First we need a stat buffer.
    if(0 != stat(FileName.c_str(), &FileNameStat)) {                            // If we can't get the stat we
        return 0;                                                               // will return 0;
    }                                                                           // If all goes well we return
    return FileNameStat.st_mtime;                                               // the last modified time_t.
}

void snf_Reloader::captureFileStats() {                                         // Get stats for later comparison.
    snfCFGData& C = *(MyRulebase.MyCFGmgr.ActiveConfiguration());               // Reference the active config.
    RulebaseFileCheckName = C.RuleFilePath;                                     // Build/Get Rulebase File Name.
    ConfigFileCheckName = C.ConfigFilePath;                                     // Build/Get Configuration File Name.
    IgnoreListCheckFileName = C.paths_workspace_path;                           // Build/Get Ignore File Name.
    IgnoreListCheckFileName.append("GBUdbIgnoreList.txt");
    RulebaseFileTimestamp = getFileTimestamp(RulebaseFileCheckName);            // Timestamps to check for
    ConfigurationTimestamp = getFileTimestamp(ConfigFileCheckName);             // changes in configuration data
    IgnoreListTimestamp = getFileTimestamp(IgnoreListCheckFileName);            // or rulebase files.
}

bool snf_Reloader::StatsAreDifferent() {                                        // Check file stats for changes.
    return (                                                                    // Return true if any of the
      RulebaseFileTimestamp != getFileTimestamp(RulebaseFileCheckName) ||       // Rulebase File, or the
      ConfigurationTimestamp != getFileTimestamp(ConfigFileCheckName) ||        // Configuration File, or the
      IgnoreListTimestamp != getFileTimestamp(IgnoreListCheckFileName)          // Ignore List File have changed.
    );
}

const int MSPerSec = 1000;                                                      // 1000 milliseconds per second.

void snf_Reloader::captureGetterConfig() {                                      // Update RulebaseGetter config.
    snfCFGData& C = *(MyRulebase.MyCFGmgr.ActiveConfiguration());               // Reference the active config.
    RulebaseGetterIsTurnedOn = (                                                // Is the script caller on or off?
      true == C.update_script_on_off &&                                         // We're on if the bit is set and
      0 < C.update_script_call.length()                                         // we have a non-empty script to call.
    );
    if(RulebaseGetterIsTurnedOn) {                                              // If it is turned on:
        RulebaseGetter.SystemCall(C.update_script_call);                        // Set the script call and
        RulebaseGetter.GuardTime(C.update_script_guard_time * MSPerSec);        // the cycle guard time.
    }
    else {                                                                      // If the scripter is turned off:
        RulebaseGetter.SystemCall("");                                          // Set the script to nothing.
    }
}

const std::string snfReloadContext = "--RELOADING--";                           // Context for info and error logs.

void snf_Reloader::myTask() {                                                   // How do we do this refresh thing?
    cd::Sleeper WaitATic(1000);                                                 // Wait a second between checks.
    while(!TimeToStop) {                                                        // While it's not time to stop:

        if(
          RulebaseGetterIsTurnedOn &&                                           // If our rulebase getter is enabled
          MyRulebase.MyLOGmgr.isUpdateAvailable()                               // and a new rulebase is availalbe:
          ) {
            RulebaseGetter.trigger();                                           // Trigger the update script (if any).
        }

        if(StatsAreDifferent()) {                                               // Check the stats. If different:
            try {                                                               // safely attempt a reload.
                WaitATic();                                                     // Wait a tic to let things stabilize
                MyRulebase.refresh();                                           // then call refresh on the handler.
                captureFileStats();                                             // If it works, capture the new stats.
                captureGetterConfig();                                          // Also update the RulebaseGetter.
                MyRulebase.logThisInfo(                                         // Log our success.
                  snfReloadContext, snf_SUCCESS, "Success");
            }
            catch(const snf_RulebaseHandler::IgnoreListError&) {                // If we get an IgnoreListError - say so.
                MyRulebase.logThisError(
                  snfReloadContext, snf_ERROR_RULE_FILE, "IgnoreListError");
            }
            catch(const snf_RulebaseHandler::ConfigurationError&) {             // If we get a ConfigurationError - say so.
                MyRulebase.logThisError(
                  snfReloadContext, snf_ERROR_RULE_FILE, "ConfigurationError");
            }
            catch(const snf_RulebaseHandler::FileError&) {                      // If we get a FileError - say so.
                MyRulebase.logThisError(
                  snfReloadContext, snf_ERROR_RULE_FILE, "FileError");
            }
            catch(const snf_RulebaseHandler::AuthenticationError&) {            // If we get a Auth Error - say so.
                MyRulebase.logThisError(
                  snfReloadContext, snf_ERROR_RULE_AUTH, "AuthError");
            }
            catch(const snf_RulebaseHandler::Busy&) {                           // If we get a Busy Exception - say so.
                MyRulebase.logThisError(
                  snfReloadContext, snf_ERROR_UNKNOWN, "BusyError");
            }
            catch(const snf_RulebaseHandler::Panic&) {                          // If we get a Panic - say so.
                MyRulebase.logThisError(
                  snfReloadContext, snf_ERROR_UNKNOWN, "PanicError");
            }
            catch(...) {                                                        // If we get some other error - shout!
                MyRulebase.logThisError(
                  snfReloadContext, snf_ERROR_UNKNOWN, "UnhandledError");
            }
        }
        WaitATic();                                                             // Wait before the next loop.
    }
}

const cd::ThreadType snf_Reloader::Type("snf_Reloader");                        // The thread's type.

snf_Reloader::snf_Reloader(snf_RulebaseHandler& R) :                            // When we are created, we
  Thread(snf_Reloader::Type, "Reloader"),                                       // brand and name our thread.
  MyRulebase(R),                                                                // Capture the rulebase handler.
  TimeToStop(false),                                                            // It's not time to stop yet.
  RulebaseGetter("RulebaseGetter"),                                             // Setup our ScriptCaller thread.
  RulebaseGetterIsTurnedOn(false) {                                             // Rulebase getter is off at first.
    captureFileStats();                                                         // Set up the initial stats.
    captureGetterConfig();                                                      // Set up RulebaseGetter config.
    run();                                                                      // Run our maintenenace thread.
  }

snf_Reloader::~snf_Reloader() {                                                 // When we are destroyed we
    TimeToStop = true;                                                          // set our time to stop bit
    join();                                                                     // and wait for the thread.
}

//// snfCFGPacket Methods

snfCFGPacket::snfCFGPacket(snf_RulebaseHandler* R) :                            // When we are created:
  MyRulebase(R),                                                                // Capture our rulebase handler and
  MyTokenMatrix(NULL),                                                          // ready our token matrix and
  MyCFGData(NULL) {                                                             // cfg pointers.
    if(MyRulebase) { MyRulebase->grab(*this); }                                 // Safely grab our rulebase.
}

snfCFGPacket::~snfCFGPacket() { if(MyRulebase) MyRulebase->drop(*this); }       // Safely drop our rulebase when we die.

TokenMatrix* snfCFGPacket::Tokens() { return MyTokenMatrix; }                   // Consumers read the Token Matrix and
snfCFGData* snfCFGPacket::Config() { return MyCFGData; }                        // the snfCFGData.

bool snfCFGPacket::bad() {                                                      // If anything is missing it's not good.
    return (NULL == MyTokenMatrix || NULL == MyCFGData);                        // True if any of these aren  NULL.
}

bool snfCFGPacket::isRulePanic(int R) {                                         // Test for a rule panic.
    return(RulePanics.end() != RulePanics.find(R));                             // Find it in the list, it's a panic.
}

//// Rulebase Handler Methods

snf_RulebaseHandler::~snf_RulebaseHandler(){                                    // Destruct the handler.
  close();                                                                      // Close before we go.
}

bool snf_RulebaseHandler::isReady(){                                            // Is the object ready?
  return (NULL!=Rulebase);                                                      // Have Rulebase? We're ready.
}

bool snf_RulebaseHandler::isBusy(){                                             // Is a refresh/open in progress or
  return (RefreshInProgress || 0<RetiringCount);                                // an older rulebase is not yet retired.
}

int snf_RulebaseHandler::getReferenceCount(){                                   // How many Engines using this handler.
  return ReferenceCount;                                                        // Tell them the count bob.
}

int snf_RulebaseHandler::getCurrentCount(){                                     // How many Engines active in the current rb.
  return CurrentCount;                                                          // Tell them what it is bob.
}

int snf_RulebaseHandler::getRetiringCount(){                                    // How many Engines active in the old rb.
  return RetiringCount;                                                         // Tell them what it is bob.
}

// FileUTC(FileName) - utility function for tagging the active rulebase

cd::RuntimeCheck FileUTCGoodTimestampLength("SNFMulti.cpp:FileUTC snprintf(...) == CorrectTimestampLength");

std::string FileUTC(std::string FileName) {                                     // Gets a files UTC.
    struct stat FileNameStat;                                                   // First we need a stat buffer.
    std::string t;                                                              // We also need a Timestamp holder.
    if(0 != stat(FileName.c_str(), &FileNameStat)) {                            // If we can't get the stat we
        t.append("00000000000000"); return t;                                   // will return all zeroz to
    }                                                                           // make sure we should get the file.
    struct tm FileNameTime;                                                     // Allocate a time structure.
    FileNameTime = *(gmtime(&FileNameStat.st_mtime));                           // Copy the file time to it as UTC.

    char TimestampBfr[16];                                                      // Timestamp buffer.

    size_t l = snprintf(                                                        // Format yyyymmddhhmmss
      TimestampBfr, sizeof(TimestampBfr),
      "%04d%02d%02d%02d%02d%02d",
      FileNameTime.tm_year+1900,
      FileNameTime.tm_mon+1,
      FileNameTime.tm_mday,
      FileNameTime.tm_hour,
      FileNameTime.tm_min,
      FileNameTime.tm_sec
    );

    const size_t CorrectTimestampLength = 4+2+2+2+2+2;
    FileUTCGoodTimestampLength(l == CorrectTimestampLength);

    t.append(TimestampBfr);                                                     // Append the timestamp to t
    return t;                                                                   // and return it to the caller.
}

// Auto Reload Controls

bool snf_RulebaseHandler::AutoRefresh(bool On) {                                // Turn on/off auto refresh.

    if(On) {                                                                    // If they want Reload On:
        if(!AutoRefresh()) {                                                    // and it isn't already on:
            try { MyReloader = new snf_Reloader(*this); }                       // try to set up a Reloader.
            catch(...) { MyReloader = 0; }                                      // If that fails we don't
        }                                                                       // have one. If it's already
    }                                                                           // on do nothing.

    else {                                                                      // If they want Reload Off:
        if(AutoRefresh()) {                                                     // and it is turned on:
            delete MyReloader;                                                  // destroy the reloader and
            MyReloader = 0;                                                     // zero it's pointer.
        }
    }
    return AutoRefresh();                                                       // Return the truth (on/off)
}

bool snf_RulebaseHandler::AutoRefresh() {                                       // True if AutoRefresh is on.
    return (0 != MyReloader);                                                   // If we have one, it's on.
}


// _snf_LoadNewRulebase()
// This is actually a common sub-funtion. It expects that the object is in the "RefreshInProgress" state,
// and that everything is in place and safe for a new rulebase to be loaded into the object. Once it's
// done it will reset from the "RefreshInProgress" state and along the way will throw any errors that
// are appropriate. The other functions can count on this one to polish off the various forms of rulebase
// load activity.

const cd::LogicCheck SaneRefreshProcessCheck("snf_RulebaseHandler::_snf_LoadNewRulebase():SaneRefreshProcessCheck(RefreshInProgress)");

void snf_RulebaseHandler::_snf_LoadNewRulebase(){                               // Common internal load/check routine.
  SaneRefreshProcessCheck(RefreshInProgress);                                   // We only get called when this flag is set.
  try { MyCFGmgr.load(); }                                                      // Load a fresh copy of the configuration.
  catch(...) {                                                                  // If something goes wrong:
    RefreshInProgress = false;                                                  // we are no longer "in refresh"
    throw ConfigurationError("_snf_LoadNewRulebase() MyCFGmgr.load() failed");  // throw the Configuration exception.
  }
  std::string RuleFilePath = MyCFGmgr.RuleFilePath();                           // Get our rulebase file path and our
  std::string SecurityKey = MyCFGmgr.SecurityKey();                             // security key from the CFG manager.
  if(0>=RuleFilePath.length()) {                                                // If we don't have a path, we're hosed.
    RefreshInProgress = false;                                                  // We are no longer "in refresh"
    throw FileError("_snf_LoadNewRulebase() Zero length RuleFilePath");         // Can't load a RB file with no path!
  }
  if(0>=SecurityKey.length()) {                                                 // No security string? toast!
    RefreshInProgress = false;                                                  // We are no longer "in refresh"
    throw AuthenticationError("snf_LoadNewRulebase() Zero length SecurityKey"); // Can't authenticate without a key!
  }

  // Notify sub modules of the new configuration data.

  MyGeneration++;                                                               // Increment the generation number.

  snfCFGData& CFGData = (*(MyCFGmgr.ActiveConfiguration()));                    // Capture the active config...

  CFGData.Generation = MyGeneration;                                            // Tag the configuration data.

  MyLOGmgr.configure(CFGData);                                                  // Update the LOGmgr's configuration.
  MyNETmgr.configure(CFGData);                                                  // Update the NETmgr's configuration.
  MyGBUdbmgr.configure(CFGData);                                                // Update the GBUdbmgr's configuration.

  // Load the new rulebase locally (on stack) and see if it authenticates.

  TokenMatrix* TryThis = NULL;                                                  // We need our candidate to remain in scope.

  try {                                                                         // This try block decodes the problem.
    try {                                                                       // This try block does cleanup work.
  	  TryThis = new TokenMatrix();                                              // Grab a new Token Matrix
  	  TryThis->Load(RuleFilePath);                                              // Load it from the provided file path
  	  TryThis->Validate(SecurityKey);                                           // Validate it with the provided security key
  	  TryThis->Verify(SecurityKey);                                             // Verify that it is not corrupt.
    }
    catch(...) {                                                                // Clean up after any exceptions.
      RefreshInProgress = false;                                                // We're not refreshing now.
      if(TryThis) {                                                             // If we allocated a TokenMatrix then
          delete TryThis;                                                       // we need to reclaim the memory
          TryThis = 0;                                                          // and erase the pointer.
      }                                                                         // With everything nice and clean we can
      throw;                                                                    // rethrow he exception for decoding.
    }
  }                                                                             // If nothing threw, we're golden!

  catch (const TokenMatrix::BadFile&) {                                         // BadFile translates to FileError
  	throw FileError("_snf_LoadNewRulebase() TokenMatrix::BadFile");
  }
  catch (const TokenMatrix::BadMatrix&) {                                       // BadMatrix translates to AuthenticationError
    throw AuthenticationError("_snf_LoadNewRulebase() TokenMatrix::BadMatrix");
  }
  catch (const TokenMatrix::BadAllocation&) {                                   // BadAllocation translates to AllocationError
    throw AllocationError("_snf_LoadNewRulebase() TokenMatrix::BadAllocation");
  }
  catch (const TokenMatrix::OutOfRange&) {                                      // OutOfRange should never happen so PANIC!
  	throw Panic("_snf_LoadNewRulebase() TokenMatrix::OutOfRange");
  }
  catch (...) {                                                                 // Something unpredicted happens? PANIC!
  	throw Panic("_snf_LoadNewRulebase() TokenMatrix.load() ???");
  }

  // At this point the rulebase looks good. If we need to go big-endian do it!

#ifdef __BIG_ENDIAN__

  TryThis->FlipEndian();                                                        // Flip tokens to big-endian format.

#endif

  MyLOGmgr.updateActiveUTC(FileUTC(RuleFilePath));                              // Update the Active Rulebase UTC.

  MyMutex.lock();                                                               // Lock the mutex while changing state.
  OldRulebase = Rulebase;                                                       // Move the current rulebase and count to
  RetiringCount = CurrentCount;                                                 // the retiring slot.

  if(0>=RetiringCount && NULL!=OldRulebase) {                                   // If nobody cares about the old rulebase
  	delete OldRulebase;                                                         // then delete it, and wipe everything
  	OldRulebase = NULL;                                                         // clean for the next retiree.
  	RetiringCount = 0;
  }

  CurrentCount = 0;                                                             // Set the current count to zero (it's fresh!)
  Rulebase = TryThis;                                                           // Copy our new rulebase into production.
  MyMutex.unlock();                                                             // Release the hounds!!!

  // If there is a GBUdb Ignore List, refresh with it (This might go elsewhere).
  // Failure to read the GBUdbIgnoreList if all else went well does not cause
  // the rulebase update (if any) to fail.

  /**** This section needs work ****/
  try {
      std::string IgnoreListPath = CFGData.paths_workspace_path;
      IgnoreListPath.append("GBUdbIgnoreList.txt");
      if(0 == MyGBUdb.readIgnoreList(IgnoreListPath.c_str()))                   // We must have at least 1 IP listed.
        throw ConfigurationError(
          "_snf_LoadNewRulebase() GBUdbIgnoreList min 1 entry!");
  }

  catch(...) {                                                                  // Ignore list read might fail.
      RefreshInProgress = false;                                                // If so, don't keep things hung.
      throw IgnoreListError("_snf_LoadNewRulebase() readIgnoreList() ???");     // If it does, throw FileError.
  }

  RefreshInProgress = false;                                                    // Done with the refresh process.
  return;                                                                       // Our work is done here.
}

// open()
// This loads a new rulebase (usually the first one only) into the handler. This is the first of two loading
// methods on this object. This one checks for isBusy() because it is highly invasive. If it is called after
// the object has been running it is important that it not run while anything in the object is active. This
// is because it is likely in this case we would be loading an entirely new rulebase that would lead to odd
// results if some scanner instances were activily using a different one.

void snf_RulebaseHandler::open(const char* path, const char* licenseid, const char* authentication){

  MyMutex.lock();                                                                   // Lock the mutex while changing state.
  if(isBusy()) {                                                                    // Be sure we're not busy.
    MyMutex.unlock(); throw Busy("snf_RulebaseHandler::open() busy");               // If we are then throw.
  }
  RefreshInProgress = true;                                                         // Set RefreshInProgress.
  MyMutex.unlock();                                                                 // Unlock the mutex and
  MyCFGmgr.initialize(path, licenseid, authentication);                             // Initialize our configuration.
  _snf_LoadNewRulebase();                                                           // get on with loading the rulebase.
  MyGBUdbmgr.load();                                                                // Load the GBUdb as configured.
  AutoRefresh(true);                                                                // Turn on Refresh by default.
  logThisInfo("--INITIALIZING--", 0, "Success");                                    // Log the happy event.
  return;
}

// refresh()
// This loads a fresh copy of the current rulebase. This is the second loading method on the object. It is
// specifically designed to work without stopping scanning activities. This one checks for isBusy() because
// there may be an old rulebase that is not yet completely retired --- that is, some scanners may be using it.
// If there is still an old rulebase on it's way out then we can't shove it aside without breaking something,
// so we have to throw.
//
// Under normal circumstances, this call will cause a new rulebase to be loaded without disturbing any scans
// underway on the current rulebase. The current rulebase will be put into retirement while any active scans
// are completed, and then it will quietly go away when the last has finished. The new rulebase will take it's
// place and will be handed out to all new grab() requests.

void snf_RulebaseHandler::refresh(){                                                // Reloads the rulebase.
  MyMutex.lock();                                                                   // Lock the mutex while changing states.
  if(isBusy()) {                                                                    // If we're busy then throw.
    MyMutex.unlock(); throw Busy("snf_RulebaseHandler::refresh() busy");
  }
  RefreshInProgress = true;                                                         // Set RefreshInProgress and
  MyMutex.unlock();                                                                 // unlock the mutex. Then get on with
  _snf_LoadNewRulebase();                                                           // loading a fresh copy of the rulebase
  return;
}

void snf_RulebaseHandler::close(){                                                  // Closes this handler.
  try {
    AutoRefresh(false);                                                             // Stop AutoRefresh if it's on.
  }
  catch(const std::exception& e) { throw e; }                                           // Rethrow good exceptions.
  catch(...) { throw Panic("snf_RulebaseHandler::close() AutoRefresh(false) panic!"); } // Panic blank exceptions.

  try {
    MyXCImgr.stop();                                                                // Stop the XCI manager.
  }
  catch(const std::exception& e) { throw e; }                                           // Rethrow good exceptions.
  catch(...) { throw Panic("snf_RulebaseHandler::close() MyXCImgr.stop() panic!"); }    // Panic blank exceptions.

  if(isBusy() || 0<CurrentCount || 0<ReferenceCount) {                              // Check that there is no activity.
    throw Busy("snf_RulebaseHandler::close() busy");                                // With XCI stopped we should not
  }                                                                                 // be busy.

  try {
    MyLOGmgr.stop();                                                                // Stop the LOG manager.
  }
  catch(const std::exception& e) { throw e; }                                           // Rethrow good exceptions.
  catch(...) { throw Panic("snf_RulebaseHandler::close() MyLOGmgr.stop() panic!"); }    // Panic blank exceptions.

  try {
    MyNETmgr.stop();                                                                // Stop the NET manager.
  }
  catch(const std::exception& e) { throw e; }                                           // Rethrow good exceptions.
  catch(...) { throw Panic("snf_RulebaseHandler::close() MyNETmgr.stop() panic!"); }    // Panic blank exceptions.

  try {
    MyGBUdbmgr.stop();                                                              // Stop the GBUdb manager.
  }
  catch(const std::exception& e) { throw e; }                                           // Rethrow good exceptions.
  catch(...) { throw Panic("snf_RulebaseHandler::close() MyGBUdbmgr.stop() panic!"); }  // Panic blank exceptions.

  try {
    if(NULL!=Rulebase) {delete Rulebase; Rulebase=NULL;}                            // If we have a Rulebase destroy it.
  }
  catch(const std::exception& e) { throw e; }                                           // Rethrow good exceptions.
  catch(...) { throw Panic("snf_RulebaseHandler::close() delete Rulebase panic!"); }    // Panic blank exceptions.

  try {
    if(NULL!=OldRulebase) {delete OldRulebase; OldRulebase=NULL;}                   // Shouldn't happen, but just in case.
  }
  catch(const std::exception& e) { throw e; }                                            // Rethrow good exceptions.
  catch(...) { throw Panic("snf_RulebaseHandler::close() delete OldRulebase panic!"); }  // Panic blank exceptions.
}

void snf_RulebaseHandler::use(){                                                    // Make use of this Rulebase Handler.
  MyMutex.lock();                                                                   // Lock the object
  ReferenceCount++;                                                                 // Boost the count
  MyMutex.unlock();                                                                 // Unlock the object
}

void snf_RulebaseHandler::unuse(){                                                  // Finish with this Rulebase Handler.
  MyMutex.lock();                                                                   // Lock the object
  ReferenceCount--;                                                                 // Reduce the count
  MyMutex.unlock();                                                                 // Unlock the object
}

// A word about Generation... In practice whenever the configuration or rulebase
// changes the entire thing is reloaded. The Generation() function gives other
// modules a way to know if they need to update their interpretation of the
// configuration. They can keep track of the last Generation value they got and
// compare it to the latest Generation. If the two are different then they need
// to update their configuration - just in case it has changed.

int snf_RulebaseHandler::Generation() { return MyGeneration; }                      // Returns the generation number.

// A word about autopanics.
// The first time throgh this we outsmarted ourselves with an ellaborate
// wait-to-insert scheme. That led to the possibilty of a deadlock. Now we
// copy the (usually empty or very short) set of rule panics to the
// configuration packet when it is grabbed and only use the one mutext to hold
// the configuration steady while doing so. All queries are made to the local
// copy of the panic list and all writes are made, under mutex, to the active
// configuration. Simpler, no significant penalty, and no more deadlocks.

// A word about configuration packets.
// Along the way we simplified things by making the snfCFGPacket do it's own
// grab and drop upon construction and destruction. This way we don't have to
// remember to handle all possible cases during a scan or other opertion -- once
// the operation goes out of scope the configuration packet drop()s with it.

void snf_RulebaseHandler::grab(snfCFGPacket& CP) {                              // Activate this Rulebase.
  cd::ScopeMutex HoldStillPlease(MyMutex);                                      // Lock the rulebase until we're done.
  CurrentCount++;                                                               // Boost the count for myself.
  CP.MyTokenMatrix = Rulebase;                                                  // Grab the current rulebase.
  CP.MyCFGData = MyCFGmgr.ActiveConfiguration();                                // Grab the active configuration.
  CP.RulePanics = MyCFGmgr.ActiveConfiguration()->RulePanicHandler.IntegerSet;  // Copy the RulePanic set.
}

void snf_RulebaseHandler::drop(snfCFGPacket& CP) {                              // Deactiveate this Rulebase.
  const TokenMatrix* t = CP.MyTokenMatrix;                                      // Grab the token matrix pointer.
  CP.MyCFGData = NULL;                                                          // Null the configuration pointer.
  cd::ScopeMutex HoldStillPlease(MyMutex);                                      // Lock the rulebase until we're done.
  if(t==Rulebase) {                                                             // If we're dropping the current rulebase
  	CurrentCount--;                                                             // then reduce the current count.
  } else                                                                        // If not that then...
  if(t==OldRulebase) {                                                          // If we're dropping the old rulebase
    RetiringCount--;                                                            // reduce the retiring count and check...
    if(0>=RetiringCount) {                                                      // to see if it is completely retired.
      if(NULL!=OldRulebase) delete OldRulebase;                                 // If it is then delete it and
      OldRulebase = NULL; RetiringCount = 0;                                    // reset it's pointer and counter.
    }
  } else {                                                                      // If we're dropping something else,
    throw Panic("snf_RulebaseHandler::drop() panic");                           // it is time to panic, so, then PANIC!
  }
}

// When adding a rule panic entry the rulebase and configuration state cannot
// be changed, nor grabbed by an snfCFGPacket. This ensures that the IntegerSet
// is only adjusted by one thread at a time and that any threads using the set
// will have a consistent result based on their last grab().

void snf_RulebaseHandler::addRulePanic(int RuleID) {                            // Add a rule panic id dynamically.
    cd::ScopeMutex JustMe(MyMutex);                                             // Freeze the rulebase while we adjust
    MyCFGmgr.ActiveConfiguration()                                              // the active configuration to
      ->RulePanicHandler.IntegerSet.insert(RuleID);                             // insert the new rule panic ruleid.
}                                                                               // When we're done, unlock and move on.

IPTestRecord& snf_RulebaseHandler::performIPTest(IPTestRecord& I) {             // Perform an IP test.
    snfCFGPacket MyCFGPacket(this);                                             // We need a CFG packet.
    try {                                                                       // Safely process the IP.
        if(false == MyCFGPacket.bad()) {                                        // If we've got a good packet:
            I.G = MyGBUdb.getRecord(I.IP);                                      // Lookup the IP in GBUdb.
            I.R = MyCFGPacket.Config()->RangeEvaluation(I.G);                   // Evaluate it's statistics.

            // Convert the RangeEvaluation into the configured Code

            switch(I.R) {
                case Unknown:                                                   // Unknown - not defined.
                case Normal:                                                    // Benefit of the doubt.
                case New: {                                                     // It is new to us.
                    I.Code = 0;                                                 // Zero is the default - no code.
                    break;
                }
                case White: {                                                   // This is a good guy.
                    I.Code = MyCFGPacket.Config()->WhiteRangeHandler.Symbol;
                    break;
                }
                case Caution: {                                                 // This is suspicious.
                    I.Code = MyCFGPacket.Config()->CautionRangeHandler.Symbol;
                    break;
                }
                case Black: {                                                   // This is bad.
                    I.Code = MyCFGPacket.Config()->BlackRangeHandler.Symbol;
                    break;
                }
                case Truncate: {                                                // Don't even bother looking.
                    I.Code = MyCFGPacket.Config()
                      ->gbudb_regions_black_truncate_symbol;
                    break;
                }
            }
        }                                                                       // If something is broken we punt.
    } catch (...) {}                                                            // Ignore exceptions (none expected)
    return I;                                                                   // Return the processed record.
}

void snf_RulebaseHandler::logThisIPTest(IPTestRecord& I, std::string Action) {  // Log an IP test result & action.
    MyLOGmgr.logThisIPTest(I, Action);
}

void snf_RulebaseHandler::logThisError(                                         // Log an error message.
  std::string ContextName, int Code, std::string Text
  ) {
    MyLOGmgr.logThisError(ContextName, Code, Text);
}

void snf_RulebaseHandler::logThisInfo(                                          // Log an informational message.
  std::string ContextName, int Code, std::string Text
  ) {
    MyLOGmgr.logThisInfo(ContextName, Code, Text);
}

std::string snf_RulebaseHandler::PlatformVersion(std::string NewPlatformVersion) { // Set platform version info.
    return MyLOGmgr.PlatformVersion(NewPlatformVersion);
}

std::string snf_RulebaseHandler::PlatformVersion() {                            // Get platform version info.
    return MyLOGmgr.PlatformVersion();
}

std::string snf_RulebaseHandler::PlatformConfiguration() {                      // Get platform configuration.
    cd::ScopeMutex LockAndGrab(MyMutex);                                        // Freeze things for a moment and
    return MyCFGmgr.ActiveConfiguration()->PlatformElementContents;             // copy the platform configuration.
}

std::string snf_RulebaseHandler::EngineVersion() {                              // Get engine version info.
    return MyLOGmgr.EngineVersion();
}

void snf_RulebaseHandler::
  XCIServerCommandHandler(snfXCIServerCommandHandler& XCH) {                    // Registers a new XCI Srvr Cmd handler.
    cd::ScopeMutex ThereCanBeOnlyOne(XCIServerCommandMutex);                    // Serialize access to this resource.
    myXCIServerCommandHandler = &XCH;                                           // Assign the new handler as provided.
}

std::string snf_RulebaseHandler::processXCIServerCommandRequest(snf_xci& X) {   // Handle a parsed XCI Srvr Cmd request.
    cd::ScopeMutex ThereCanBeOnlyOne(XCIServerCommandMutex);                    // Serialize access to this resource.
    if(0 == myXCIServerCommandHandler) {                                        // If we don't have a handler then
        snfXCIServerCommandHandler H;                                           // create a base handler and
        return H.processXCIRequest(X);                                          // return it's default response.
    }                                                                           // If we do have a handler then pass
    return myXCIServerCommandHandler->processXCIRequest(X);                     // on the request and return the
}                                                                               // response.

//// snf_IPTestEngine Methods

snf_IPTestEngine::snf_IPTestEngine() :                                          // The constructor is simple - it
  Lookup(NULL), ScanData(NULL) {                                                // sets up our internal references.
}                                                                               // Before use these must be set.

void snf_IPTestEngine::setGBUdb(GBUdb& G) {                                     // Here's how we set the GBUdb.
    Lookup = &G;
}

void snf_IPTestEngine::setScanData(snfScanData& S) {                            // Here's how we set the ScanData object.
    ScanData = &S;
}

void snf_IPTestEngine::setCFGData(snfCFGData& C) {                              // Here's how we set the CFGData.
    CFGData = &C;
}

void snf_IPTestEngine::setLOGmgr(snfLOGmgr& L) {                                // Here's how we set the LOGmgr.
    LOGmgr = &L;
}

// 20090127 _M Added special handling for forced IP sources. First, they are
// always considered the source and second if they are in the GBUdb ignore list
// then GBUdb training bypass is established.

std::string& snf_IPTestEngine::test(std::string& input, std::string& output) {  // Perform IP lookups and put IPs into ScanData.

    if(NULL == Lookup || NULL == ScanData) {                                    // If we are not set up properly then we
        output = "{IPTest Config Error}";                                       // will return an error string.
        return output;
    }

    try {                                                                       // If we're out of IP records, no analysis.
        IPScanRecord& I = ScanData->newIPScanRecord();                          // Grab a new IP scan record and
        cd::IP4Address IP = input;                                              // Convert the string to an IP.

        // Identify forced Source IP addresses

        bool ThisSourceIsForced = (                                             // This IP is a forced source IP if:
          (0 == I.Ordinal) && (                                                 // we are looking at the first IP and
            (0UL != ScanData->CallerForcedSourceIP()) ||                        // either the Caller forced the IP or
            (0UL != ScanData->HeaderDirectiveSourceIP())                        // the IP was forced by a header directive.
            )
        );

        // Bad IPs are possible, especially if the source was forced. In that
        // case forced source IP is meaningless so we want to ignore it and
        // we want to make the case visible in the logs. An ordinary IP that
        // is invalid has no consequence so we simply skip those.

        // Note that a source IP that has it's ignore flag set causes an
        // implied training bypass inside the scan function. Setting the bad
        // IP as the source and setting it's ignore flag will have the desired
        // effect.

        if(0UL == IP) {                                                         // If we got a 0 or a bad conversion then
            output = "{0.0.0.0 Is Not A Usable IP}";                            // we won't be testing this IP.
            if(ThisSourceIsForced) {                                            // If this ip is a forced source then
                I.GBUdbData.Flag(Ignore);                                       // we will force a training bypass,
                ScanData->SourceIPRecord(I);                                    // we will record it as the source,
                ScanData->SourceIPEvaluation = output;                          // and capture the error output.
            }
            return output;
        }

        if(0xFFFFFFFF == IP) {                                                  // If we got a 255.255.255.255 then
            output = "{255.255.255.255 Is Not A Usable IP}";                    // we won't be testing this IP.
            if(ThisSourceIsForced) {                                            // If this ip is a forced source then
                I.GBUdbData.Flag(Ignore);                                       // we will force a training bypass,
                ScanData->SourceIPRecord(I);                                    // we will record it as the source,
                ScanData->SourceIPEvaluation = output;                          // and capture the error output.
            }
            return output;
        }

        GBUdbRecord R = Lookup->getRecord(IP);                                  // Get the GBUdb record for it.
        I.IP = IP;                                                              // store the IP and the
        I.GBUdbData = R;                                                        // GBUdb record we retrieved.


        output = "{";                                                           // Next we start to build our IP data insert.
        std::ostringstream S;                                                   // We will use a string stream for formatting.
        switch(R.Flag()) {                                                      // Identify the flag data for this IP.
            case Good: S << "Good "; break;
            case Bad:  S << "Bad "; break;
            case Ugly: S << "Ugly "; break;
            case Ignore: S << "Ignore "; break;
        }
        S << "c=" << R.Confidence() << " "                                      // Include the Confidence and
          << "p=" << R.Probability();                                           // Probability.

        // Process ordinary Source IP addresses

        if(                                                                     // The message source IP address is the
          (false == ScanData->FoundSourceIP()) &&                               // first IP we find that is either forced
          (ThisSourceIsForced || (Ignore != R.Flag()))                          // OR is NOT part of our infrastructure.
          ) {                                                                   // When we find the correct source IP:

            if(                                                                 // Check to see if we're drilling down.
              (false == ThisSourceIsForced) &&                                  // We drill when the source is NOT forced
              (ScanData->isDrillDownSource(I))                                  // AND we have a matching drilldown.
              ) {
                Lookup->setIgnore(IP);                                          // If we're drilling down ignore this IP.
            }
            else {                                                              // If not drilling down this is the source:

                ScanData->SourceIPRecord(I);                                    // we log it in as the source
                S << " Source";                                                 // and report our findings in our tag.

                // Since we are dealing with our source IP
                // this is a good place to evaluate our truncate feature.

                snfIPRange IPR =
                  ScanData->SourceIPRange(CFGData->RangeEvaluation(R));         // Establish the IP range for this scan.

                // We will also emit a range identifier for pattern matches that might use it.

                switch(IPR) {
                    case Unknown:   { S << " Unknown"; break; }                 // Unknown - not defined.
                    case White:     { S << " White"; break; }                   // This is a good guy.
                    case Normal:    { S << " Normal"; break; }                  // Benefit of the doubt.
                    case New:       { S << " New"; break; }                     // It is new to us.
                    case Caution:   { S << " Caution"; break; }                 // This is suspicious.
                    case Black:     { S << " Black"; break; }                   // This is bad.
                    case Truncate:  { S << " Truncate"; break; }                // Don't even bother looking.
                }

                ScanData->SourceIPEvaluation = S.str();                         // Capture the source IP eval.

                // The RangeEvaluation() call above settles a lot of questions for us.
                // The Truncate return code only happens when the IP is either Bad w/
                // truncate turned on, or the statistics place the IP in the Truncate
                // range. If the Good flag is set the function always returns White so
                // here we only have to check for the Truncate flag.

                if(Truncate == IPR) {                                           // If all of the conditions are met
                    ScanData->GBUdbTruncateTriggered = true;                    // then Truncate has been triggered.
                    ScanData->GBUdbPeekTriggered = LOGmgr->OkToPeek(            // Since truncate was triggered, see if
                      CFGData->gbudb_regions_black_truncate_peek_one_in);       // we would also trigger a peek.

                    // The reason we check the truncate on_off flag here is that the
                    // IP range _may_ return a Truncate result if no Flags are set on
                    // the IP and the IP is far enough into the black to reach the
                    // Truncate threshold.

                    if(CFGData->gbudb_regions_black_truncate_on_off) {          // If truncate is on either peek or truncate.
                        if(ScanData->GBUdbPeekTriggered) {                      // If a peek has been triggered then
                            ScanData->GBUdbPeekExecuted = true;                 // mark the event and don't truncate.
                        } else {                                                // If a peek was not triggered then
                            ScanData->GBUdbTruncateExecuted = true;             // Record our trucnate action.
                            output = "";                                        // Set up the truncate signal (empty string)
                            return output;                                      // and return it! We're done!
                        }
                    }
                }
            }
        }

        // If we're not truncating then we're going to return our IP evaulation tag
        // to the filter chain function module so it can emit it into the stream.

        output.append(S.str());
        output.append("}");
    }
    catch(snfScanData::NoFreeIPScanRecords) {
        output = "{too_many}";
    }
    catch(...) {
        output = "{fault}";
    }
    return output;
}

//// Engine Handler Methods

snf_EngineHandler::~snf_EngineHandler(){                                        // Shutdown clenas up and checks for safety.
  if(isReady()) close();                                                        // If we're live, close on our way out.
}

void snf_EngineHandler::open(snf_RulebaseHandler* Handler){                     // Light up the engine.
  MyMutex.lock();                                                               // Serialize this...
  if(isReady()) {                                                               // If we're already open then we need to
    MyMutex.unlock();                                                           // unlock this object and let them know
    throw Busy("snf_EngineHandler::open() busy");                               // we are busy.
  }                                                                             // If we're not busy, then let's light it up.
  MyRulebase=Handler;                                                           // Install our rulebase handler.
  MyRulebase->use();                                                            // Up the use count to let it know we're here.
  MyIPTestEngine.setGBUdb(MyRulebase->MyGBUdb);                                 // Set up the IPTester's GBUdb.
  MyIPTestEngine.setScanData(MyScanData);                                       // Set up the IPTester's ScanData reference.
  MyIPTestEngine.setLOGmgr(MyRulebase->MyLOGmgr);                               // Set up the IPTester's LOGmgr.
  MyMutex.unlock();                                                             // Unlock our mutex, then...
  return;                                                                       // our work is done.
}

bool snf_EngineHandler::isReady(){                                              // Is the Engine good to go?
  return (NULL!=MyRulebase);                                                    // Have rulebase will travel.
}

void snf_EngineHandler::close(){                                                // Close down the engine.
  MyMutex.lock();                                                               // Serialize this...
  if(!isReady()){                                                               // If we're not already open we can't close.
    MyMutex.unlock();                                                           // Something is seriously wrong, so unlock
    throw Panic("snf_EngineHandler::close() !isReady panic");                   // and hit the panic button!
  }                                                                             // But, if everything is ok then we can
  MyRulebase->unuse();                                                          // unuse our rulebase and quietly forget
  MyRulebase = NULL;                                                            // about it.
  if(NULL!=CurrentMatrix) {                                                     // If we have a leftover evaluation matrix
    delete CurrentMatrix;                                                       // we can let that go and forget about
    CurrentMatrix = NULL;                                                       // it as well.
  }
  MyMutex.unlock();                                                             // Finally, we unlock our mutex and...
  return;                                                                       // Our work is done here.
}

enum PatternResultTypes {                                                       // To train GBUdb we need a generalized
    NoPattern,                                                                  // way to evaluate the results from the
    WhitePattern,                                                               // snf pattern matching scan.
    BlackPattern,
    IPPattern,
    AboveBandPattern
};

// In order to optimize message file reads when header injection is not activated
// we need to look ahead to see if header injection is likely to be turned on when
// we do the scan. This is a short term fix. The better fix might be to perform
// the configuration load prior to scanning the message -- but that is a much larger
// refactoring that ties up configuration and rulebase resources for a longer time.
// Instead we're going to take an optimistic route and just peek at the configuration.
// If the configuration changes while we're loading the file to be scanned then
// we have two cases. If we go from XHDRInject off to XHDRInject on then we will
// miss adding headers to the message - not a bad outcome. If we go from XHDRInject
// on to XHDRInject off then we might emit headers for an extra message - also not
// a bad outcome.

bool snf_RulebaseHandler::testXHDRInjectOn() {
  cd::ScopeMutex HoldStillPlease(MyMutex);                                      // Lock the rulebase until we're done.
  snfCFGData* myCFG = MyCFGmgr.ActiveConfiguration();                           // Grab the active configuration.
  bool myXHDRInjectOnFlag = (LogOutputMode_Inject == myCFG->XHDROutput_Mode);   // True if output mode is inject.
  return myXHDRInjectOnFlag;                                                    // return the result.
}

int snf_EngineHandler::scanMessageFile(                                         // Scan this message file.
  const std::string MessageFilePath,                                            // -- this is the file path (and id)
  const int MessageSetupTime,                                                   // -- setup time already used.
  const cd::IP4Address MessageSource                                            // -- message source IP (for injection).
  ) {

    cd::Timer AdditionalSetupTime;
    cd::ScopeMutex DoingAFileScan(FileScan);                                    // Protect MyScanData @ this entry.

    // Preliminary setup. Clearing the ScanData resets the ReadyToClear flag
    // and allows us to set some data for more accurate tracking and so that if
    // something goes wrong the ScanData will be helpful in determining the
    // state of the engine.

    MyScanData.clear();                                                         // Clear the scan data.
    MyScanData.StartOfJobUTC = MyRulebase->MyLOGmgr.Timestamp();                // Set the job start timestamp.
    MyScanData.ScanName = MessageFilePath;

    // Now that the preliminaries are established we can begin our work.

    int MessageFileSize = 0;                                                    // Here will be the size of it.
    std::ifstream MessageFile;                                                  // Here will be our input file.
    MessageFile.exceptions(                                                     // It will throw exceptions for
      std::ifstream::eofbit | std::ifstream::failbit | std::ifstream::badbit    // these unwanted events.
    );

    try {                                                                       // Try opening the message file.
        MessageFile.open(MessageFilePath.c_str(), 
          std::ios::in | std::ios::binary);                                     // Open the file, binary mode.

        MessageFile.seekg(0, std::ios::end);                                    // Find the end of the file,
        MessageFileSize = MessageFile.tellg();                                  // read that position as the size,
        MessageFile.seekg(0, std::ios::beg);                                    // then go back to the beginning.
        MyScanData.ScanSize = MessageFileSize;                                  // Capture the message file size.
    }
    catch(...) {                                                                // Trouble? Throw FileError.
        MyRulebase->MyLOGmgr.logThisError(                                      // Log the error.
          MyScanData, "scanMessageFile().open",
          snf_ERROR_MSG_FILE, "ERROR_MSG_FILE"
        );
        throw FileError("snf_EngineHandler::scanMessageFile() Open/Seek");
    }

    if(0 >= MessageFileSize) {                                                  // Handle zero length files.
        MessageFile.close();                                                    // No need to keep this open.
        MyRulebase->MyLOGmgr.logThisError(                                      // Log the error.
          MyScanData, "scanMessageFile().isFileEmpty?",
          snf_ERROR_MSG_FILE, "ERROR_MSG_FILE"
        );
        throw FileError("snf_EngineHandler::scanMessageFile() FileEmpty!");
    }

    bool isXHeaderInjectionOn = MyRulebase->testXHDRInjectOn();
    bool noNeedToReadFullFile = (false == isXHeaderInjectionOn);
    if(noNeedToReadFullFile) {
        MessageFileSize = std::min(MessageFileSize, snf_ScanHorizon);
    }

    std::vector<unsigned char> MessageBuffer;                                   // Allocate a buffer and size
    try { MessageBuffer.resize(MessageFileSize, 0); }                           // it to fit the message.
    catch(...) {                                                                // Trouble? Throw AllocationError.
        MyRulebase->MyLOGmgr.logThisError(                                      // Log the error.
          MyScanData, "scanMessageFile().alloc",
          snf_ERROR_MSG_FILE, "ERROR_MSG_ALLOC"
        );
        throw AllocationError("snf_EngineHandler::scanMessageFile() Alloc");
    }

    try { MessageFile.read((char*) &MessageBuffer[0], MessageFileSize); }       // Read the file into the buffer.
    catch(...) {
        MyRulebase->MyLOGmgr.logThisError(                                      // Log the error.
          MyScanData, "scanMessageFile().read",
          snf_ERROR_MSG_FILE, "ERROR_MSG_READ"
        );
        throw FileError("snf_EngineHandler::scanMessageFile() Read");
    }
    MessageFile.close();                                                        // Close the file.

    // Additional Setup Time will be captured as the call is made.

    int ScanResultCode = scanMessage(                                           // Scan the message we've loaded.
      &MessageBuffer[0],                                                        // Here is the buffer pointer,
      MessageBuffer.size(),                                                     // here is the size of the message,
      MessageFilePath,                                                          // the path is the identifier,
      (AdditionalSetupTime.getElapsedTime() + MessageSetupTime),                // and this is our setup time total.
      MessageSource                                                             // Pass on the source if provided.
    );

    // Inject headers if required.

    if(isXHeaderInjectionOn) {                                                  // If we are to inject headers:
        const char* XHDRInjStage = "Begin";                                     // Keep track of what we're doing.
        try {

            // The insertion point will be at the end of the existing headers.
            // We pick that point to be right between the two <cr><lf> so that
            // the first blank line will appear at the end of our headers.
            // We accommodate either <cr><lf> or <lf> line endings.
            // We are careful not to search past the end of unreasonably short
            // message files.

            unsigned int InsertPoint = 0;                                       // Find the insertion point.
            bool UseLFOnly = false;                                             // Use \n line endings in files?
            bool CRLFPresent = false;                                           // Detected \r\n pairs?

            unsigned int BiggestPatternSize = 4;                                // How far we look ahead.
            bool BigEnoughMessage = BiggestPatternSize < MessageBuffer.size();

            if(BigEnoughMessage){
                unsigned int Limit = MessageBuffer.size() - BiggestPatternSize;
                bool DataWasSkipped = MessageBuffer.size() > MyScanData.ScanSize;

                unsigned int i = 0;
                if(DataWasSkipped) {                                            // If our scanner skipped data at
                    i = MessageBuffer.size() - MyScanData.ScanSize;             // the top of the message buffer then
                }                                                               // we will skip it too.

                for(; i < Limit; i++) {                                         // Search for the first blank line.

                    if(                                                         // Detect CRLF pairs if present.
                      false == CRLFPresent &&
                      '\r' == MessageBuffer.at(i) &&
                      '\n' == MessageBuffer.at(i + 1)
                      ) CRLFPresent = true;

                    if(                                                         // In a properly formatted RFC822
                      '\r' == MessageBuffer.at(i) &&                            // message that looks like
                      '\n' == MessageBuffer.at(i + 1) &&                        // <cr><lf><cr><lf>
                      '\r' == MessageBuffer.at(i + 2) &&
                      '\n' == MessageBuffer.at(i + 3)
                      ) {
                        InsertPoint = i + 2;
                        break;
                    } else
                    if(                                                         // In some bizarre cases it might
                      '\n' == MessageBuffer.at(i) &&                            // look like <lf><lf>.
                      '\n' == MessageBuffer.at(i + 1)
                      ) {
                        InsertPoint = i + 1;
                        UseLFOnly = true;                                       // We have to strip <CR> from our
                        break;                                                  // injected header line ends.
                    }
                }
            }

            // Here we must interpret the results of our search. Do we know where
            // our insert point is or do we punt and use the top of the message?

            if(0 == InsertPoint) {                                              // No blank line? We need to punt.
                if(false == CRLFPresent) {                                      // What kind of line ends do we use?
                    UseLFOnly = true;                                           // If no CRLF found use LF only.
                }                                                               // Either way we will be inserting
            }                                                                   // our headers at the top of the msg.

            // At this point we know where to split the message and insert
            // our X Headers.

            XHDRInjStage = "Open Temp File";                                    // Update our process monitor.

            std::string TempFileName = MessageFilePath;                         // Prepare a temp file name
            TempFileName.append(".tmp");                                        // based on the message file.

            std::ofstream TempFile;                                               // Here will be our temp file.
            TempFile.exceptions(std::ofstream::failbit | std::ofstream::badbit);  // It will throw these exceptions.
            TempFile.open(TempFileName.c_str(), 
              std::ios::binary | std::ios::trunc);                              // Open and truncate the file.

            // If our insert point is the top of the message we'll skip this.

            if(0 < InsertPoint) {                                               // If we have an insert point:
                XHDRInjStage = "Write Temp File.1";                             // Update our process monitor.
                TempFile.write(                                                 // Write the message file up
                  reinterpret_cast<char*>(&MessageBuffer[0]),                   // to our split.
                  InsertPoint
                );
            }

            // If our file has \n line ends we need to strip the \r from our
            // rfc822 \r\n line ends.

            XHDRInjStage = "XHDR <CR><LF> to <LF>";

            if(true == UseLFOnly) {                                             // If we are using <LF> only:
                std::string ReworkedHeaders = "";                               // Make a new string and rework
                for(                                                            // our headers.
                  std::string::iterator iS = MyScanData.XHDRsText.begin();      // Run through the headers one
                  iS != MyScanData.XHDRsText.end(); iS++                        // byte at a time.
                  ) {
                    if('\r' != (*iS)) ReworkedHeaders.push_back(*iS);           // Strip out any <CR> chars.
                }
                MyScanData.XHDRsText.swap(ReworkedHeaders);                     // Swap in our reworked headers.
            }

            // Now we are ready to inject our headers.

            XHDRInjStage = "Write Temp File.2";                                 // Update our process monitor.

            TempFile.write(                                                     // Inject our headers.
              MyScanData.XHDRsText.c_str(),
              MyScanData.XHDRsText.length()
            );

            XHDRInjStage = "Write Temp File.3";                                 // Update our process monitor.

            TempFile.write(                                                     // Write the rest of the message.
              reinterpret_cast<char*>(&MessageBuffer[InsertPoint]),
              MessageBuffer.size() - InsertPoint
            );

            XHDRInjStage = "Close Temp File";                                   // Update our process monitor.

            TempFile.close();                                                   // Close the file (flushing it).

            cd::Sleeper PauseBeforeRetry(300);                                      // Delay to use between retries.

            XHDRInjStage = "Drop Msg";                                          // Update our process monitor.

            if(remove(MessageFilePath.c_str())) {                               // Remove the old message file
                PauseBeforeRetry();                                             // If it fails, pause and retry.
                if(remove(MessageFilePath.c_str())) {                           // If that fails,
                    PauseBeforeRetry();                                         // pause, then try once more.
                    if(remove(MessageFilePath.c_str())) {                       // If that fails, throw.
                        throw XHDRError("XHDR injector can't remove original!");
                    }
                }
            }

            XHDRInjStage = "Rename Temp -> Msg";                                // Update our process monitor.

            if(rename(TempFileName.c_str(), MessageFilePath.c_str())) {         // Make Temp our new message file.
                PauseBeforeRetry();                                             // If it fails, pause and retry.
                if(rename(TempFileName.c_str(), MessageFilePath.c_str())) {     // If that fails,
                    PauseBeforeRetry();                                         // pause then try once more.
                    if(rename(TempFileName.c_str(), MessageFilePath.c_str())) { // If that fails, throw.
                        throw XHDRError("XHDR injector can't rename tmp file!");
                    }
                }
            }
        }
        catch(XHDRError& e) {                                                   // For full XHDRError exceptions.
            std::string ERROR_MSG_XHDRi = "ERROR_MSG_XHDRi: ";                  // Format the XHDRInj error msg.
            ERROR_MSG_XHDRi.append(XHDRInjStage);
            ERROR_MSG_XHDRi.append(" ");
            ERROR_MSG_XHDRi.append(e.what());
            MyRulebase->MyLOGmgr.logThisError(                                  // Log the error.
              MyScanData, "scanMessageFile().xhdr.inject",
              snf_ERROR_MSG_FILE, ERROR_MSG_XHDRi
            );
            throw;                                                              // Rethrow any XHDRError exceptions.
        }
        catch(const std::exception& e) {                                        // For ordinary runtime exceptions.
            std::string ERROR_MSG_XHDRi = "ERROR_MSG_XHDRi: ";                  // Format the XHDRInj error msg.
            ERROR_MSG_XHDRi.append(XHDRInjStage);
            ERROR_MSG_XHDRi.append(" ");
            ERROR_MSG_XHDRi.append(e.what());
            MyRulebase->MyLOGmgr.logThisError(                                  // Log the error.
              MyScanData, "scanMessageFile().xhdr.inject",
              snf_ERROR_MSG_FILE, ERROR_MSG_XHDRi
            );
            throw XHDRError(ERROR_MSG_XHDRi);                                   // Rethrow as XHDRError exceptions.
        }
        catch(...) {                                                            // If we encounter a problem then
            std::string ERROR_MSG_XHDRi = "ERROR_MSG_XHDRi: ";                  // Format the XHDRInj error msg.
            ERROR_MSG_XHDRi.append(XHDRInjStage);
            MyRulebase->MyLOGmgr.logThisError(                                  // Log the error.
              MyScanData, "scanMessageFile().xhdr.inject",
              snf_ERROR_MSG_FILE, ERROR_MSG_XHDRi
            );
            std::string XHDRError_msg = "Message Rewrite Failed: ";             // Format our throw message with
            XHDRError_msg.append(XHDRInjStage);                                 // our detailed stage data and
            throw XHDRError(XHDRError_msg);                                     // throw our special exception.
        }
    }

    // Create an .xhdr file if required.

    if(MyScanData.XHeaderFileOn) {
        try {
            std::ofstream XHDRFile;                                               // Output file will be XHDRFile.
            XHDRFile.exceptions(std::ofstream::failbit | std::ofstream::badbit);  // These events will throw exceptions.
            std::string XHDRFileName = MessageFilePath;                           // Build the XHDR file name by adding
            XHDRFileName.append(".xhdr");                                         // .xhdr to the message file name.
            XHDRFile.open(XHDRFileName.c_str(), 
              std::ios::binary | std::ios::trunc);                              // Open (and truncate) the file.
            XHDRFile << MyScanData.XHDRsText;                                   // Spit out the XHDRs.
            XHDRFile.close();                                                   // All done.
        }
        catch(...) {                                                            // If we encounter a problem then
            MyRulebase->MyLOGmgr.logThisError(                                  // Log the error.
              MyScanData, "scanMessageFile().xhdr.file",
              snf_ERROR_MSG_FILE, "ERROR_MSG_XHDRf"
            );
            throw XHDRError(".xhdr file write failed");                         // throw our special exception.
        }
    }

    return ScanResultCode; // Return the actual result, of course.
}

std::string snf_EngineHandler::extractMessageID(                                // Find and return the first Message-ID
  const unsigned char* Msg,                                                     // Input the Message buffer to search
  const int Len                                                                 // and the length of the buffer.
  ) {
    std::string ExtractedID = "";                                               // Start with an empty string.
    bool FoundID = false;                                                       // Haven't found it yet.
    int C = 0;                                                                  // Cursor position.
    while(!FoundID && (C < (Len - 12))) {                                       // Loop through the Msg looking for
      if(                                                                       // the Message-ID: header.
        ('\n' == Msg[C]) &&                                                     // Starting at the new line find
        ('M' == Msg[C +  1] || 'm'  == Msg[C +  1]) &&                          // Message-ID: (per RFC822)
        ('e' == Msg[C +  2] || 'E'  == Msg[C +  2]) &&
        ('s' == Msg[C +  3] || 'S'  == Msg[C +  3]) &&                          // We use an unrolled comparison
        ('s' == Msg[C +  4] || 'S'  == Msg[C +  4]) &&                          // loop here for raw speed and
        ('a' == Msg[C +  5] || 'A'  == Msg[C +  5]) &&                          // optimization. Note that we
        ('g' == Msg[C +  6] || 'G'  == Msg[C +  6]) &&                          // compare the most likely characters
        ('e' == Msg[C +  7] || 'E'  == Msg[C +  7]) &&                          // first in each case, and we don't
        ('-' == Msg[C +  8]) &&                                                 // need to go through a buffer length
        ('I' == Msg[C +  9] || 'i'  == Msg[C +  9]) &&                          // check at each byte for partial
        ('D' == Msg[C + 10] || 'd'  == Msg[C + 10]) &&                          // matches.
        (':' == Msg[C + 11]) &&
        (' ' == Msg[C + 12] || '\t' == Msg[C + 12])
        ) {
          C = C + 13;                                                           // Starting just after the space
          while(C < Len) {                                                      // and staying within bounds
              unsigned char X = Msg[C];                                         // grab each character in the ID.
              if(isprint(X)) {                                                  // If it is printable,
                  if(' ' == X) X = '_';                                         // massage out the spaces as _ and
                  if(127 < X) X = '|';                                          // high characters as | and
                  if('\'' == X || '\"' == X) X = '`';                           // ' or " to ` in order to make the
                  ExtractedID.push_back(X);                                     // ID safe for logging, then push
              } else                                                            // the result into our string. When
              if('\r' == X || '\n' == X) break; /* leave copy loop */           // we reach the end we're done.
              ++C;                                                              // else get ready for the next byte.
          }
          FoundID = true;                                                       // Set the flag: we found Message-ID:
          break; /* leave search loop */                                        // We got what we came for. Break!

      } else {                                                                  // When we don't find the Message-ID:
          if(                                                                   // we check for end of headers.
            ('\n' == Msg[C] && '\n' == Msg[C+1]) ||                             // Either <LF><LF> or
            ('\r' == Msg[C] && '\n' == Msg[C+1] &&                              // <CR><LF><CF><LF>
             '\r' == Msg[C+2] && '\n' == Msg[C+3])
            ) {                                                                 // If we've found the end of headers
              break;                                                            // we're done looking. If we did not
          }                                                                     // find the end of headers then
          ++C;                                                                  // we move to the next position.
      }
    }

    // At this point we either have the Extracted ID, or we need a substitute.

    if(0 == ExtractedID.length()) {                                             // If we need a substitute ID then
        MyRulebase->MyLOGmgr.SerialNumber(ExtractedID);                         // use the next available serial number.
    }

    return ExtractedID;                                                         // Return the extracted id or substitute.
}

const cd::LogicFault FaultBadMessageBuffer1("snf_EngineHandler::scanMessage():FaultBadMessageBuffer1(NULL == inputMessageBuffer)");
const cd::LogicFault FaultBadMessageBuffer2("snf_EngineHandler::scanMessage():FaultBadMessageBuffer2(0 >= inputMessageLength)");

const char Unknown_SNFMatchFlag = '-';
const char Panic_SNFMatchFlag = 'p';
const char Match_SNFMatchFlag = 'm';
const char White_SNFMatchFlag = 'w';
const char Final_SNFMatchFlag = 'f';

void captureMatchRecord(snf_match& M, MatchRecord* R) {
  M.flag = Unknown_SNFMatchFlag;
  M.ruleid = R->RuleId();
  M.symbol = R->RuleGroup();
  M.index = R->MatchStartPosition;
  M.endex = R->MatchEndPosition;
}

void snf_SaccadesHandler::applySaccades(EvaluationMatrix* Scanner, std::vector<unsigned char>& Data) {
    if(NULL == Scanner) return;

    bool isTimeToPeek = (0 >= TimeToPeekCounter);
    if(isTimeToPeek) {
        TimeToPeekCounter = TimeToPeekReset;
        return;
    } else {
        --TimeToPeekCounter;
    }

    std::vector<saccade> Saccades = grabSaccades();
    for(std::vector<saccade>::iterator i = Saccades.begin(); i != Saccades.end(); i++) {
        const saccade& s = (*i);
        if(s.start >= Data.size()) break;
        Scanner->evaluateSegment(Data, s.start, s.finish);
    }
}

bool isLearnableMatch(MatchRecord* m) {
    bool isGoodSymbol = (0 <= m->RuleGroup() && 64 > m->RuleGroup());
    bool isBeyondAlwaysScan = (snf_SaccadesHandler::AlwaysScanLength < m->MatchEndPosition);
    return (isGoodSymbol && isBeyondAlwaysScan);
}

void snf_SaccadesHandler::learnMatches(MatchRecord* Matches) {
    if(NULL == Matches) return;

    std::vector<saccade> MatchesToLearn;
    saccade WatchForHeaderWhiteRules(0, AlwaysScanLength);
    MatchesToLearn.push_back(WatchForHeaderWhiteRules);

    for(MatchRecord* m = Matches; NULL != m; m = m->NextMatchRecord) {
        if(isLearnableMatch(m)) {
            MatchesToLearn.push_back(
              saccade(
                m->MatchStartPosition,
                m->MatchEndPosition)
            );
        }
    }

    if(0 < MatchesToLearn.size()) {
        lockAndLearn(MatchesToLearn);
    }
}

static snf_SaccadesHandler SaccadeBrain;
static snf_IPStrangerList StrangersList;

int snf_EngineHandler::scanMessage(                                             // Scan this message (in buffer).
  const unsigned char* inputMessageBuffer,                                      // -- this is the message buffer.
  const int inputMessageLength,                                                 // -- this is the length of the buffer.
  const std::string MessageName,                                                // -- this is the message identifier.
  const int MessageSetupTime,                                                   // -- setup time used (for logging).
  const cd::IP4Address MessageSource                                            // -- message source IP (for injection).
  ) {

  cd::ScopeTimer ScanTimeCapture(MyScanData.ScanTime);                          // Start the scan time clock.

  unsigned char* MessageBuffer = NULL;                                          // Explicitly initialize these two
  int MessageLength = 0;                                                        // so the compiler will be happy.

  FaultBadMessageBuffer1(NULL == inputMessageBuffer);                           // Fault on null message buffer.
  FaultBadMessageBuffer2(0 >= inputMessageLength);                              // Fault on bad message bfr length.

  // Protect this engine - only one scan at a time per EngineHandler ;-)

  cd::ScopeMutex ScannerIsBusy(MyMutex);                                        // Serialize this...

  // Preliminary job setup.

  // In our pre-processing we may adjust our input buffer so we capture the
  // originals and then use the captured values. For example if we are scanning
  // Communigate message files we will want to skip the communigate headers.

  MessageBuffer = const_cast<unsigned char*>(inputMessageBuffer);               // Capture the input buffer.
  MessageLength = inputMessageLength;                                           // Capture the input length.

  MyScanData.clear();                                                           // Clear the scan data.
  MyScanData.ScanSize = MessageLength;                                          // Grab the message length.
  MyScanData.SetupTime = MessageSetupTime;                                      // Capture the setup time.

  if(0 == MyScanData.StartOfJobUTC) {                                           // If the job timestamp is not
    MyScanData.StartOfJobUTC = MyRulebase->MyLOGmgr.Timestamp();                // yet set then set it.
  }

  MyScanData.CallerForcedSourceIP(MessageSource);                               // Capture the MessageSource if any.

  // Special note about exceptions here...
  // Setting up the filter chain can throw an exception. It can't go in it's own try block or it will
  // be out of scope for the remainder of the function... SO, I've wrapped everything inside of the
  // Lock() in a try block ... and there's a nested one also for scanning the content. The result is
  // that I can put all of the unlock work in the "outer" try block and re-throw anything that's
  // needed.

  snfCFGPacket MyCFGPacket(MyRulebase);                                         // We need this to stay in scope.

  // Set up the filter chain, configure the scanner, and scan the message.

  try {

    if(MyCFGPacket.bad()) {                                                     // If it's not there it's a big problem.
        throw Panic("snf_EngineHandler::scanMessage() MyCFGPacket.bad()");
    }

    // Adapt to CGP message files - skip the CGP headers

    MyScanData.MessageFileTypeCGPOn =                                           // Find out if we are expecting
      MyCFGPacket.Config()->MessageFileTypeCGP_on_off;                          // Communigate message files.

    if(MyScanData.MessageFileTypeCGPOn) {                                       // If we are scanning CGP files:
        while(4 < MessageLength) {                                              // Skip over the CGP headers.
            if(                                                                 // On Winx systems look for the first
              '\r' == MessageBuffer[0] &&                                       // blank line encoded as CRLF CRLF.
              '\n' == MessageBuffer[1] &&
              '\r' == MessageBuffer[2] &&
              '\n' == MessageBuffer[3]
              ) {                                                               // If we find it then skip past
                MessageBuffer += 4;                                             // the new line and break out
                MessageLength -= 4;                                             // of the loop.
                break;
            } else                                                              // On *nix systems look for the first
            if(                                                                 // blank line encoded as LF LF.
              '\n' == MessageBuffer[0] &&
              '\n' == MessageBuffer[1]
              ) {                                                               // If we find it then skip past
                MessageBuffer += 2;                                             // the blank line and break out
                MessageLength -= 2;                                             // of the loop.
                break;
            }
            else {                                                              // If we don't find it then
                ++MessageBuffer;                                                // eat one byte from the buffer
                --MessageLength;                                                // and keep going.
            }
        }

        // At this point our MessagBuffer contains just the message we
        // want to scan.

        MyScanData.ScanSize = MessageLength;                                    // Reset the scan size.
    }

    // Identify this message.

    if(                                                                         // How do we identify this scan?
      0 == MessageName.length() ||                                              // If no name was provided or
      true == MyCFGPacket.Config()->Scan_Identifier_Force_Message_Id            // we are forcing RFC822 IDs then
      ) {                                                                       // extract the Message-ID from the
        MyScanData.ScanName = extractMessageID(MessageBuffer, MessageLength);   // message and use that.
    } else {                                                                    // If a name was provided and we
        MyScanData.ScanName = MessageName;                                      // are not forcing RFC822 IDs then
    }                                                                           // use the name provided to us.

    // Set up our filter chain.

    std::stringstream PrependedHeaders;                                         // Use this to prepend X-Headers.
    FilterChainCBFR IU(MessageBuffer, MessageLength, PrependedHeaders);         // Set up the filter chain.
    FilterChainHeaderAnalysis IV(&IU, MyIPTestEngine);                          // Include header analysis.
    FilterChainBase64 IW(&IV);                                                  // Include Base64 decoding.
    FilterChainQuotedPrintable IX(&IW);                                         // Include Quoted Printable decoding.
    FilterChainUrlDecode IY(&IX);                                               // Include URL decoder.
    FilterChainDefunker IZ(&IY);                                                // Include Defunking.

    // Now we set up our scanner and grab the current token matrix.

    if(NULL!=CurrentMatrix) { delete CurrentMatrix; CurrentMatrix=NULL; }       // If we have old results, delete them.
    try {
        CurrentMatrix = new EvaluationMatrix(MyCFGPacket.Tokens());             // Allocate a new matrix for this scan.
    } catch(...) {                                                              // Check that the allocation worked.
        throw AllocationError("new EvaluationMatrix() ???");
    }

    // Here we get down to it and start scanning the message.

    const char* DebugInfo = "scanMessage() Begin Message Scan";                 // If we panic, here we are.

    try {

      // The IPTestEngine has the ability to truncate the message in the filter
      // chain under certain conditions. In order to configure those conditions
      // the IPTestEngine needs to have the configuration data being used for
      // the current scan.

      DebugInfo = "scanMessage() setCFGData()";                                 // If we panic, here we are.

      MyIPTestEngine.setCFGData(*(MyCFGPacket.Config()));                       // Setup the CFG data to use.

      // Check processed headers for header directive rules. One of these might
      // include a directive to get the message source IP from a header. If so
      // then MyScanData will have been modified. Also if there are drill-down
      // directives then MyScanData will have been modified to mark any headers
      // that should be ignored -- in this case the IP test used in the filter
      // chain will take appropriate action as it comes across the Received
      // headers that have been marked.

      DebugInfo = "scanMessage() Get Header Directives";

      MyScanData.HeaderDirectiveFlags = 0x00000000;                             // Clear the header directive flags.
      if(0 < MyCFGPacket.Config()->                                             // Check to see if we have any
        HeaderDirectivesHandler.HeaderDirectives.size()) {                      // header directive rules and if we do:
        HeaderFinder HeaderDirectivesParser(                                    // Parse the headers in the message
          &MyScanData,                                                          // and update the ScanData using the
          MyCFGPacket.Config()->HeaderDirectivesHandler.HeaderDirectives,       // directives in our configuration packet.
          MessageBuffer,                                                        // Pass the message as a pointer with
          MessageLength                                                         // a specific buffer length.
        );
        MyScanData.HeaderDirectiveFlags = HeaderDirectivesParser();             // Capture the parsed results.
      }

      // Message header rules in earlier versions occasionally failed because there was not
      // a new-line character in front of the very first header. So, now we insert one :-)
      // This allows all header rules to start off with a ^ indicating the start of the line.

      // 20070719_M Added \n to X-snfScanSize: synthetic header.

      // 20070120_M There are some messages where the size is a specific part of
      // the pattern so we will now be emitting this data into the engine. A later
      // version of the engine should handle this kind of thing using a special
      // filter chain module.

      DebugInfo = "scanMessage() ^X-snfScanSize";                               // If we panic here we are.
                                                                                // Build the scan size info
      PrependedHeaders << "X-snfScanSize: " << MyScanData.ScanSize << "\n";     // and format as an X- header.

      // Add a phantom received header to the top IF the message source has been
      // forced by the caller or by a header directive. After that the normal
      // scanning and header analysis process should pick up the IP as the
      // source of the message. (It will not if the IP is ignored in the GBUdb!)

      DebugInfo = "scanMessage() PhantomReceived";                              // If we panic we are here.

      if(0UL != MyScanData.CallerForcedSourceIP()) {                            // If the caller forced the source IP:
        PrependedHeaders                                                        // Make a phantom Received header
          << "Received: Caller.Forced.Source.IP ["                              // showing that the caller forced
          << (std::string) MyScanData.CallerForcedSourceIP() << "]\n";          // the source IP.
      } else
                                                                                // If not forced by the caller but a
      if(0UL != MyScanData.HeaderDirectiveSourceIP()) {                         // header directive forced the source IP:
        PrependedHeaders                                                        // Make a phantom Received header
          << "Received: Header.Directive.Source.IP ["                           // showing that a header directive
          << (std::string) MyScanData.HeaderDirectiveSourceIP() << "]\n";       // established the source IP.
      }

      // Most of the time we will extract the source IP the normal way.

      // If there are other prepended headers to add they should go here.

      /** Add other prepended headers **/

      // 20070719_M Reworked the engine to handle the filter-chain section in
      // a tight loop separately from the scanning section. This should allow
      // for tighter optimization in some cases (less cache thrashing) and also
      // provides for later development of parallel analysis of the pre-filtered
      // data, as well as the ability to output the pre-filtered data for use in
      // rule development and debugging.

      DebugInfo = "scanMessage() IZ.GetByte() ==> FilteredData";                // If we panic we are here.
      unsigned char xb=0;
      MyScanData.FilteredData.clear();                                          // Clear the FilteredData buffer.
      try {                                                                     // Watch for exceptions and scan
          for(int a = 0; a < snf_ScanHorizon; a++)                              // the message through the filter
              MyScanData.FilteredData.push_back(xb=IZ.GetByte());               // chain into the FilteredData buffer.
      }                                                                         // When we run out of data we will
      catch(const FilterChain::Empty&) {}                                       // get the Empty exception and stop.

      // Scan each byte in the file up to the horizon or the end of the message.
      // If something goes wrong, an exception will be thrown.

      DebugInfo = "scanMessage() EvaluateThis(FilteredData)";                   // If we panic, here we are.

      if(false == MyScanData.GBUdbTruncateExecuted) {                           // If we haven't already truncated:
          //for(int a = 0, b = MyScanData.FilteredData.size(); a < b; a++)        // Scan through the filtered data one
          //  CurrentMatrix->EvaluateThis(MyScanData.FilteredData[a]);            // byte at a time.

          /** 20200618_M Experiment stripping out saccades to check performance hit
          unsigned int fullLength = MyScanData.FilteredData.size();

          SaccadeBrain.applySaccades(CurrentMatrix, MyScanData.FilteredData);
          bool messageNotRecognized = (NULL == CurrentMatrix->ResultList);
          if(messageNotRecognized) {
              CurrentMatrix->evaluateSegment(MyScanData.FilteredData, 0, fullLength);
              SaccadeBrain.learnMatches(CurrentMatrix->ResultList);
          }
          **/
        size_t fullLength = MyScanData.FilteredData.size();
        CurrentMatrix->evaluateSegment(MyScanData.FilteredData, 0, fullLength); // Only do a full scan -- see comment above
      }

      DebugInfo = "scanMessage() Scan Data Complete";                           // If we panic, here we are.
    }
    catch(const EvaluationMatrix::BadAllocation&) {                             // Check for bad allocation during scan.
        throw AllocationError("EvaluationMatrix::BadAllocation");
    }
    catch(const EvaluationMatrix::MaxEvalsExceeded&) {                          // Check for too many evaluators.
        throw MaxEvals("EvaluationMatrix::MaxEvalsExceeded");
    }
    catch(const EvaluationMatrix::OutOfRange&) {                                // Check for out of range of (bad) matrix.
        throw BadMatrix("EvaluationMatrix::OutOfRange");
    }
    catch(...){                                                                 // In order to prevent thread craziness
      throw Panic(DebugInfo);                                                   // throw a Panic.
    }                                                                           // The mutex will unlock in the outer try.
  }

  // Here is the end of the outer try block. We can catch and rethrow whatever happend
  // and we can also keep our mutex properly managed.

  catch(AllocationError& e) {                                                   // Allocation Errors pass through.
    MyRulebase->MyLOGmgr.logThisError(                                          // Log the error.
      MyScanData, "scanMessage()",
      snf_ERROR_ALLOCATION, "ERROR_ALLOCATION"
    );
    throw;
  }
  catch(MaxEvals& e) {                                                          // MaxEvals == Panic, with a log.
    MyRulebase->MyLOGmgr.logThisError(                                          // Log the error.
      MyScanData, "scanMessage()",
      snf_ERROR_MAX_EVALS, "ERROR_MAX_EVALS"
    );
    throw;
  }
  catch(BadMatrix& e) {                                                         // BadMatrix == Panic, with a log.
    MyRulebase->MyLOGmgr.logThisError(                                          // Log the error.
      MyScanData, "scanMessage()",
      snf_ERROR_BAD_MATRIX, "ERROR_BAD_MATRIX"
    );
    throw;
  }
  catch(Panic& e) {                                                             // Panic is panic.
    MyRulebase->MyLOGmgr.logThisError(                                          // Log the error.
      MyScanData, "scanMessage()",
      snf_ERROR_BAD_MATRIX, "ERROR_PANIC"
    );
    throw;
  }
  catch(const std::exception& e) {                                              // Other exceptions.
    MyRulebase->MyLOGmgr.logThisError(                                          // Log the error.
      MyScanData, "scanMessage()",
      snf_ERROR_UNKNOWN, "ERROR_EXCEPTION"
    );
    throw;
  }
  catch(...) {                                                                  // Anything else == Panic.
    MyRulebase->MyLOGmgr.logThisError(                                          // Log the error.
      MyScanData, "scanMessage()",
      snf_ERROR_UNKNOWN, "ERROR_UNKNOWN"
    );
    throw Panic("snf_EngineHandler::scanMessage() ERROR_UNKNOWN!");
  }

  // At this point, we've completed our scan and we're ready to evaluate our results to find the correct symbol to return.

  ResultsCount = 0;                                                             // Reset the count,
  ResultsRemaining = 0;                                                         // Remaining count,
  FinalResult = NULL;                                                           // Final Result marker,
  ResultCursor = CurrentMatrix -> ResultList;                                   // And cursor position for our results.

  // Now that our result processing gadgets are reset, let's process the results list.

  int const CLEAN_RESULT = 0;                                                   // CLEAN means no matches or white.
  int const NO_SYMBOL = 999;                                                    // NO_SYMBOL is higher than any SYMBOL
  int S = NO_SYMBOL;                                                            // so we start there and work down.

  snf_match TmpSNFMatch;                                                        // We'll need a buffer for our matches.

  while(NULL!=ResultCursor) {                                                   // While we have records to process...
    captureMatchRecord(TmpSNFMatch, ResultCursor);                              // grab the next record and evaluate it.

    // Mitigate short-match rulebase events to prevent false positives.

    const size_t minimumPatternLength = 5;                                      // Establish a minimum match length.
    size_t matchSpan = (TmpSNFMatch.endex - TmpSNFMatch.index);                 // Determine the length of this match.
    bool isShortMatchEvent = (minimumPatternLength > matchSpan);                // Identify short-match events.

    bool isPanickedRule = (                                                     // In addition to rule IDs that are
      MyCFGPacket.isRulePanic(TmpSNFMatch.ruleid) ||                            // in the rule-panic list also treat
      isShortMatchEvent                                                         // short match events as panic rules.
    );

    bool isVotingCandidate = (false == isPanickedRule);                         // Panic rules can't vote.

    bool isWhiteRule = (
      MyCFGPacket.Config()->TrainingWhiteRuleHandler.isListed(TmpSNFMatch.ruleid) ||
      0 == TmpSNFMatch.symbol
    );
    bool isBestResultCode = (TmpSNFMatch.symbol < S);

    // Set an appropriate flag.

    if(isPanickedRule) TmpSNFMatch.flag = Panic_SNFMatchFlag;
    else if(isWhiteRule) TmpSNFMatch.flag = White_SNFMatchFlag;
    else TmpSNFMatch.flag = Match_SNFMatchFlag;

    // Vote for best rule match.

    if(isVotingCandidate && isBestResultCode) {
        FinalResult = ResultCursor;
        S = TmpSNFMatch.symbol;
    }

    // Record this MatchRecord and mMove on to next result.

    MyScanData.MatchRecords.push_back(TmpSNFMatch);
    ResultsCount++;
    ResultCursor=ResultCursor->NextMatchRecord;
  }

  if(NO_SYMBOL != S) {                                                          // If a pattern match was detected then
      MyScanData.PatternWasFound = true;                                        // trip the flag and record the
      MyScanData.PatternID = FinalResult->RuleId();                             // Rule ID and the
      MyScanData.PatternSymbol = FinalResult->RuleGroup();                      // Symbol.
  }

  //// GBUdb Integration ///////////////////////////////////////////////////////

  // To integrate GBUdb we need to generalize the result from the pattern scan.

  PatternResultTypes ScanResultType = NoPattern;                                // What kind of result have we here?
  if(0 < (MyScanData.HeaderDirectiveFlags & HeaderDirectiveWhite)) {            // If a white header directive matched
      ScanResultType = WhitePattern;                                            // then we have a "WhitePattern'.
  } else
  if(MyCFGPacket.Config()->TrainingWhiteRuleHandler.isListed(S)) {              // If the pattern was mapped to a white
      ScanResultType = WhitePattern;                                            // rule group then we have a 'WhitePattern'.
  } else
  if(CLEAN_RESULT == S) {                                                       // If there was a standard white rule
      ScanResultType = WhitePattern;                                            // result then we have a 'WhitePattern'.
  } else
  if(NO_SYMBOL == S) {                                                          // If there was no pattern match then
      ScanResultType = NoPattern;                                               // we have 'NoPattern'.
  } else
  if(63 == S) {                                                                 // If the pattern was a standard IP rule
      ScanResultType = IPPattern;                                               // then we have an 'IPPattern'.
  } else
  if(62 >= S) {                                                                 // In general, other nonzer rule groups
      ScanResultType = BlackPattern;                                            // indicate we have a 'BlackPatter'.
  } else
  if(63 < S) {                                                                  // Any pattern number > 63 is special.
      ScanResultType = AboveBandPattern;                                        // Any of these are an 'AboveBandPattern'
  }

  if(MyScanData.FoundSourceIP()) {                                              // We need an identified IP source.

      // Train the GBUdb based on our pattern matching results.

      // Evaluate our training conditions.

      bool TrainingIsTurnedOn = MyCFGPacket.Config()->GBUdbTrainingOn_Off;
      bool MessageWasNotTruncated = (false == MyScanData.GBUdbTruncateExecuted);
      bool ThereIsNoBypassHeaderDirective = (0 == (MyScanData.HeaderDirectiveFlags & HeaderDirectiveBypass));
      bool ThereIsNoBypassResultCodeRule = (false == MyCFGPacket.Config()->TrainingBypassRuleHandler.isListed(S));
      bool ThereIsNoImpliedBypassDirective = (Ignore != (MyScanData.SourceIPRecord().GBUdbData.Flag()));

      // If these conditions are favorable then train the GBUdb.

      if(                                                                       // Check to see if training is enabled.
        TrainingIsTurnedOn &&                                                   // If it is turned on AND
        MessageWasNotTruncated &&                                               // The message was not truncated AND
        ThereIsNoBypassHeaderDirective &&                                       // There is NO Bypass header directive AND
        ThereIsNoBypassResultCodeRule &&                                        // There is NO Bypass result code rule AND
        ThereIsNoImpliedBypassDirective                                         // There is NO Implied bypass directive:
        ) {

        // GBUdb training is enabled.

        bool discoveredNewIP = false;
        cd::IP4Address theSourceIP = MyScanData.SourceIPRecord().IP;

        switch(ScanResultType) {                                                // Evaluate the scan result.
            case NoPattern:                                                     // On no pattern (benefit of doubt) or
            case WhitePattern: {                                                // a white pattern:

                GBUdbRecord thisRecord =                                        // Grab the GBUdb record for later
                  MyRulebase->MyGBUdb.addGood(                                  // then add a good count to the
                    theSourceIP);                                               // source IP.

                discoveredNewIP = (0 == thisRecord.Bad() && 1 == thisRecord.Good());

                if(discoveredNewIP) {                                           // New IPs are strangers.
                    StrangersList.addStranger(theSourceIP);                     // Add them to the list
                    thisRecord.Bad(thisRecord.Good());                          // and set their reputation
                    MyRulebase->MyGBUdb.setRecord(theSourceIP, thisRecord);     // to 50/50 at best.

                } else
                if(                                                             // Known IPs that are getting
                  thisRecord.Good() > thisRecord.Bad() &&                       // an advantage but are on the
                  StrangersList.isStranger(theSourceIP)                         // strangers list get put back
                  ) {                                                           // to a 50/50 reputation.
                    unsigned int equalizationValue = thisRecord.Good();
                    if(1 < equalizationValue) equalizationValue = equalizationValue / 2;
                    thisRecord.Bad(equalizationValue);
                    thisRecord.Good(equalizationValue);
                    MyRulebase->MyGBUdb.setRecord(theSourceIP, thisRecord);
                }
                break;
            }

            case BlackPattern: {                                                // On a black pattern:

                GBUdbRecord thisRecord =                                        // Grab the GBUdb record for later
                  MyRulebase->MyGBUdb.addBad(                                     // Add a bad count to the source IP
                    MyScanData.SourceIPRecord().IP);                              // in the GBUdb.

                discoveredNewIP = (1 == thisRecord.Bad() && 0 == thisRecord.Good());
                if(discoveredNewIP) StrangersList.addStranger(theSourceIP);

                break;
            }
            default: break;                                                     // In all other cases, don't train.
        }
      }

      // GBUdb Training Is Complete

      // At this point our SourceIPRange tells us exactly how to evaluate
      // the source IP for this message.

      switch(MyScanData.SourceIPRange()) {

        case White: {                                                           // If the IP was in the white zone
          MyScanData.GBUdbWhiteTriggered = true;                                // mark that down.
          if(MyCFGPacket.Config()->WhiteRangeHandler.On_Off) {                  // If we're also turned on then
            if(                                                                 // do we need to force the symbol?
              BlackPattern == ScanResultType ||                                 // We do if the pattern scan resulted
              IPPattern == ScanResultType                                       // in a black or IPblack match.
              ) {                                                               // If we must force a white result:
                S = MyCFGPacket.Config()->WhiteRangeHandler.Symbol;             // force the symbol and
                MyScanData.GBUdbWhiteSymbolForced = true;                       // record that it was done.
            }

            // AutoPanic

            int AutoPanicRangeLowerBound =                                      // Calculate the current lower bound
              MyRulebase->MyLOGmgr.LatestRuleID() -                             // for rule id's that are eligible to
              MyCFGPacket.Config()->gbudb_regions_white_panic_rule_range;       // trigger auto-panics.

            if(BlackPattern == ScanResultType || IPPattern == ScanResultType) { // Was there a pattern/source conflict?
              MyScanData.GBUdbPatternSourceConflict = true;                     // Record the event.
              if(MyScanData.PatternID > AutoPanicRangeLowerBound) {             // If the pattern ID is in range then
                MyScanData.GBUdbAutoPanicTriggered = true;                      // record that the AutoPanic triggered.
                if(MyCFGPacket.Config()->gbudb_regions_white_panic_on_off) {    // If rule panics are turned on then
                  MyScanData.GBUdbAutoPanicExecuted = true;                     // indicate we are executing an autopanic.
                  MyRulebase->addRulePanic(MyScanData.PatternID);               // Add the rule panic.
                }
              }
            }
          }
          break;
        }

        case Normal: {                                                          // If the IP is normal...
          MyScanData.GBUdbNormalTriggered = true;                               // Count the event.
          break;                                                                // That's all.
        }

        case New: {
          break;
        }

        case Caution: {                                                         // If the IP is in the caution range.
          MyScanData.GBUdbCautionTriggered = true;                              // Track that this range fired.
          if(
            MyCFGPacket.Config()->CautionRangeHandler.On_Off &&                 // If we're also turned on and there
            NoPattern == ScanResultType                                         // is no pattern match then
            ) {                                                                 // we will override the scan result:
              S = MyCFGPacket.Config()->CautionRangeHandler.Symbol;             // set the symbol as configured and
              MyScanData.GBUdbCautionSymbolForced = true;                       // record that it was done.
          }
          break;
        }

        // Truncate is a kind of uber-black, so we do some weirdness here.
        // If Truncate happens, then black was triggered by definition. In
        // peek cases or if Truncate is turned off then Truncate might not
        // execute-- when that happens we need to fall back to Black behavior.

        case Truncate:                                                          // If the IP was in the truncate range
        case Black: {                                                           // and/or If the IP is in the black range
          MyScanData.GBUdbBlackTriggered = true;                                // mark that down.

          if(MyScanData.GBUdbTruncateExecuted) {                                // If the truncate action was executed
              S = MyCFGPacket.Config()->gbudb_regions_black_truncate_symbol;    // we set the output symbol accordingly.
          } else                                                                // Truncate overrides black.. but if
          if(                                                                   // Black is in charge do this...
            MyCFGPacket.Config()->BlackRangeHandler.On_Off &&                   // If black action is turned on and there
            NoPattern == ScanResultType                                         // is no pattern match then
            ) {                                                                 // we will override the scan data:
              S = MyCFGPacket.Config()->BlackRangeHandler.Symbol;               // set the symbol as configured and
              MyScanData.GBUdbBlackSymbolForced = true;                         // record that it was done.
          }

          // Now that all of the overrides have been handled we can handle
          // sampling. When a black IP is detected and a pattern match is not
          // then we may sample the data.

          int BlackSampleRate =                                                 // Grab the sample rate to make the
            MyCFGPacket.Config()->gbudb_regions_black_sample_grab_one_in;       // logic clearer.

          bool SampleThresholdReached =                                         // Check the spam probability of the
            (MyCFGPacket.Config()->gbudb_regions_black_sample_probability <=    // source IP against the configuration
              MyScanData.SourceIPRecord().GBUdbData.Probability());             // to see if this IP is a candidate.

          if(                                                                   // Should we sample?
            false == MyScanData.GBUdbTruncateExecuted &&                        // If this was not a truncation and
            NoPattern == ScanResultType &&                                      // No pattern match was found and
            SampleThresholdReached &&                                           // We reached out sample threshold and
            MyRulebase->MyLOGmgr.OkToSample(BlackSampleRate)                    // It's ok for us to sample this round
            ) {                                                                 // then our sampling mechanism is triggerd.
              MyScanData.GBUdbSampleTriggered = true;                           // Mark down that event.
              if(MyCFGPacket.Config()->gbudb_regions_black_sample_on_off) {     // If sampling is turned on then
                MyScanData.GBUdbSampleExecuted = true;                          // we will be sampling this data.
                if(MyCFGPacket.Config()->gbudb_regions_black_sample_passthrough) {  // If sampling by passthrough then
                    S = MyCFGPacket.Config()->                                      // Force the symbol value to passthrough
                      gbudb_regions_black_sample_passthrough_symbol;                // (usually 0 - same as CLEAN).
                } else {                                                            // If sampling internally then
                    MyRulebase->MyNETmgr.sendSample(                                // send this message as a sample.
                      (*(MyCFGPacket.Config())),                                    // Pass our current config info,
                      MyScanData,                                                   // our scan data,
                      MessageBuffer,                                                // and the message itself.
                      MessageLength
                    );
                }
              }
          }
          break;
        }

        case Unknown:                                                           // Unknown - most likely we couldn't
        default: {                                                              // find a usable source.
            break;                                                              // Do nothing.
        }
      }
  } // End of IP source depended work (GBUdbOverrides)

  // At this point we know the final result of our scan
  // and the number of results we have. It's time to set up our result
  // processing widgets for further query and return the result of this scan.

  ResultCursor = CurrentMatrix -> ResultList;                                   // Starting at the top of the list
  ResultsRemaining = ResultsCount;                                              // with all of the results ahead of us.

  if(NO_SYMBOL==S) S = CLEAN_RESULT;                                            // When there were no results, CLEAN
  MyScanData.CompositeFinalResult = S;                                          // Record what we will return.

  if(                                                                           // Prepare our final result.
    CLEAN_RESULT == S &&                                                        // If we have a clean result code
    ScanResultType != WhitePattern &&                                           // and it wasn't forced by a white
    false == MyScanData.GBUdbWhiteSymbolForced) {                               // rule or white GBUdb then we mark
      TmpSNFMatch.flag = 'c';                                                   // the final record Clean.
  } else {                                                                      // Otherwise we mark the final record
      TmpSNFMatch.flag = 'f';                                                   // as Final - meaning deliberately zero.
  }

  TmpSNFMatch.index = 0;                                                        // Our index is charater zero.
  TmpSNFMatch.endex = CurrentMatrix->CountOfCharacters - 1;                     // Our endex is the end of the message.
  TmpSNFMatch.symbol = MyScanData.CompositeFinalResult;                         // Our symbol is in CompositeFinal.

  // The rule id is dependent on what's happened...

  if(                                                                           // If the symbol has been forced...
    MyScanData.GBUdbTruncateExecuted ||                                         // Was it a Truncate-IP scan?
    MyScanData.GBUdbWhiteSymbolForced ||                                        // Was it a White-IP scan?
    MyScanData.GBUdbBlackSymbolForced ||                                        // Was it a Black-IP scan?
    MyScanData.GBUdbCautionSymbolForced ||                                      // Was it a Caution-IP scan?
    NULL == FinalResult                                                         // OR there was no valid match
    ) {                                                                         // then our rule id will be
      TmpSNFMatch.ruleid = 0;                                                   // ZERO.
  } else {                                                                      // Normally the rule id will be
      TmpSNFMatch.ruleid = FinalResult->RuleId();                               // that of the winning pattern match.
  }

  MyScanData.MatchRecords.push_back(TmpSNFMatch);                               // Push our final entry onto the list.

  MyScanData.MatchRecordsCursor = MyScanData.MatchRecords.begin();              // Reset the delivery system to the
  MyScanData.MatchRecordsDelivered = 0;                                         // beginning of the results list.

  MyScanData.ScanDepth = CurrentMatrix->MaximumCountOfEvaluators;               // Capture the scan depth.

  MyScanData.ScanTime.stop();                                                   // Stop the scan time clock.
  MyRulebase->MyLOGmgr.logThisScan((*(MyCFGPacket.Config())), MyScanData);      // Log the data from this scan.

  // Since V2-9rc19 of this engine, the Engine mutex and snfCFGPacket handle
  // their own cleanup when this call goes out of scope. ScannerIsBusy(MyMutex)
  // will unlock() on destruction and snfCFGPacket will MyRulebase->drop().

  return S;                                                                     // Return the final scan result.
}



int snf_EngineHandler::getResults(snf_match* MatchBuffer){                      // Get the next match buffer.
    cd::ScopeMutex SerializeThis(MyMutex);                                      // Serialize this...
    if(NULL == MatchBuffer) {                                                   // If we were given the reset signal
        MyScanData.MatchRecordsCursor = MyScanData.MatchRecords.begin();        // Move the cursor to the beginning
        MyScanData.MatchRecordsDelivered = 0;                                   // and reset the delivered count.
    } else {                                                                    // If we are in delivery mode and
        if(MyScanData.MatchRecords.end() != MyScanData.MatchRecordsCursor) {    // there are more to deliver then
            (*MatchBuffer) = (*MyScanData.MatchRecordsCursor);                  // deliver the current match and
            ++MyScanData.MatchRecordsCursor;                                    // move on to the next. Be sure to
            ++MyScanData.MatchRecordsDelivered;                                 // count this one as delivered.
        }
    }
    return MyScanData.MatchRecords.size() - MyScanData.MatchRecordsDelivered;   // Return a count of unseen records.
}

int snf_EngineHandler::getDepth(){                                              // Get the scan depth.
  cd::ScopeMutex SerializeThis(MyMutex);                                        // Protect our reading.
  return MyScanData.ScanDepth;                                                  // Return the latest scan depth.
}

const std::string snf_EngineHandler::getClassicLog() {                          // Get classic log entries for last scan.
    cd::ScopeMutex SerializeThis(MyMutex);                                      // Serialize this...
    return MyScanData.ClassicLogText;                                           // Return the log text.
}

const std::string snf_EngineHandler::getXMLLog() {                              // Get XML log entries or last scan.
    cd::ScopeMutex SerializeThis(MyMutex);                                      // Serialize this...
    return MyScanData.XMLLogText;                                               // Return the log text.
}

const std::string snf_EngineHandler::getXHDRs() {                               // Get XHDRs for last scan.
    cd::ScopeMutex SerializeThis(MyMutex);                                      // Serialize this...
    return MyScanData.XHDRsText;                                                // Return the XHeaders text.
}

//// Multi Engine Handler Methods

// snf_RoundRulebaseCursor()
// Returns the next rulebase slot id wrapping around to zero.

int snf_MultiEngineHandler::RoundRulebaseCursor(){                                  // Return the next Rulebase handle
  RulebaseCursor++;                                                                 // Increase the cursor.
  if(snf_MAX_RULEBASES<=RulebaseCursor)                                             // If we've reached the end of the array
    RulebaseCursor=0;                                                               // then we start back at zero.
  return RulebaseCursor;                                                            // Return the new handle candidate.
}

// snf_RoundEngineCursor()
// Returns the next engine slot id wrapping around to zero.

int snf_MultiEngineHandler::RoundEngineCursor(){                                    // Return the next Engine handle candidate.
  EngineCursor++;                                                                   // Increase the cursor.
  if(snf_MAX_SCANNERS<=EngineCursor)                                                // If we've reached the end of the array
    EngineCursor=0;                                                                 // then we start back at zero.
  return EngineCursor;                                                              // Return the new handle candidate.
}

snf_MultiEngineHandler::~snf_MultiEngineHandler(){                                  // Clean up, safety check, shut down.

  RulebaseScan.lock();                                                              // Lock both the rulebase and
  EngineScan.lock();                                                                // engine scan rulebases.

  RulebaseCursor = EngineCursor = SHUTDOWN;                                         // Set the cursors to the FINISHED value.

  // The handlers in the arrays will all get closed by their destructors.
  // The SHUTDOWN value in the cursors will force any errant threads to get no love.

  RulebaseScan.unlock();
  EngineScan.unlock();

}

// snf_OpenRulebase()
// Grab the first available rulebse handler and light it up.

int snf_MultiEngineHandler::OpenRulebase(const char* path, const char* licenseid, const char* authentication){
  RulebaseScan.lock();                                                              // Serialize this.
  if(SHUTDOWN==RulebaseCursor) {                                                    // Not ok to open after shutdown.
    RulebaseScan.unlock();
    throw Panic("snf_MultiEngineHandler::OpenRulebase() No open after shutdown");
  }

  int Handle = RoundRulebaseCursor();                                               // Grab the next hanlder on the list.
  if(RulebaseHandlers[Handle].isReady()) {                                          // Check to see if it's already in use. If so,
    int wherewasi = Handle;                                                         // keep track of where we started.
    while(RulebaseHandlers[(Handle=RoundRulebaseCursor())].isReady()){              // Loop to find an free handler.
      if(wherewasi==Handle) {                                                       // If we get back where we started
      	RulebaseScan.unlock();                                                      // Unlock the Rulebase Scanning process
      	throw TooMany("snf_MultiEngineHandler::OpenRulebase() Too Many Open");                                                            // and tell the caller Too Many are open.
      }
    }
  }

  // Now we have a Handle to a free RulebaseHandler. Time to open it up.

  try {
    RulebaseHandlers[Handle].open(path,licenseid,authentication);                   // Try to open the handler.
  }                                                                                 // If an exception is thrown...
  catch(snf_RulebaseHandler::AuthenticationError& e)                                // Catch and re-throw the appropriate
    { RulebaseScan.unlock(); throw AuthenticationError(e.what()); }                 // exception.
  catch(snf_RulebaseHandler::AllocationError& e)
    { RulebaseScan.unlock(); throw AllocationError(e.what()); }
  catch(snf_RulebaseHandler::FileError& e)
    { RulebaseScan.unlock(); throw FileError(e.what()); }
  catch(snf_RulebaseHandler::Busy& e)
    { RulebaseScan.unlock(); throw Panic(e.what()); }                               // Wasn't busy above!! Shoudn't be here!!!
  catch(const std::exception& e)
    { RulebaseScan.unlock(); throw e; }
  catch(...) {
    RulebaseScan.unlock();
    throw Panic("snf_MultiEngineHandler::OpenRulebase() ???");
  }

  RulebaseScan.unlock();                                                            // If everything went well then UnLock
  return Handle;                                                                    // and return the happy new handle.
}

// snf_RefreshRulebase()
// Reload the rulebase associated with the handler.

void snf_MultiEngineHandler::RefreshRulebase(int RulebaseHandle){                   // Refreshing a rulebase (Not Serialized)
  try {
    RulebaseHandlers[RulebaseHandle].refresh();                                     // Try to refresh the rulebase.
  }                                                                                 // Catch and rethrow any exceptions.
  catch(snf_RulebaseHandler::AuthenticationError& e) {
      throw AuthenticationError(e.what());
  }
  catch(snf_RulebaseHandler::AllocationError& e) {
      throw AllocationError(e.what());
  }
  catch(snf_RulebaseHandler::FileError& e) {
      throw FileError(e.what());
  }
  catch(snf_RulebaseHandler::Busy& e) {
      throw Busy(e.what());
  }
  catch(const std::exception& e) {
      throw e;
  }
  catch(...) {
      throw Panic("snf_MultiEngineHandler::RefreshRulebase() ???");
  }
}

// snf_CloseRulebase()
// Shut down this Rulebase handler.

void snf_MultiEngineHandler::CloseRulebase(int RulebaseHandle){                     // Closing a rulebase handler
  RulebaseScan.lock();                                                              // Serialize this - the handler changes state.
  try {                                                                             // Try to close the handler.
    RulebaseHandlers[RulebaseHandle].close();
  }
  catch(snf_RulebaseHandler::Busy& e) {                                             // A busy throw we can understand.
    RulebaseScan.unlock(); throw Busy(e.what());
  }
  catch(const std::exception& e) {                                                             // Other exceptions? rethrow.
    RulebaseScan.unlock(); throw e;
  }
  catch(...) {                                                                      // Any other throw is big trouble.
    RulebaseScan.unlock();
    throw Panic("snf_MultiEngineHandler::CloseRulebase() ???");
  }
  RulebaseScan.unlock();                                                            // When done, unlock the Rulebase Scan process.
}

// snf_OpenEngine()
// Grab the first available Engine handler and light it up

int snf_MultiEngineHandler::OpenEngine(int RulebaseHandle){

  EngineScan.lock();                                                                // Serialize this.
  if(SHUTDOWN==EngineCursor) {                                                      // Not ok to open after shutdown.
    EngineScan.unlock();
    throw Panic("snf_MultiEngineHandler::OpenEngine() No open after shutdwon");
  }

  int Handle = RoundEngineCursor();                                                 // Grab the next hanlder on the list.
  if(EngineHandlers[Handle].isReady()) {                                            // Check to see if it's already in use. If so,
    int wherewasi = Handle;                                                         // keep track of where we started.
    while(EngineHandlers[(Handle=RoundEngineCursor())].isReady()){                  // Loop to find an free handler.
      if(wherewasi==Handle) {                                                       // If we get back where we started
      	EngineScan.unlock();                                                        // Unlock the Rulebase Scanning process
      	throw TooMany("snf_MultiEngineHandler::OpenEngine() too many open");        // and tell the caller Too Many are open.
      }
    }
  }

  // Now we have a Handle to a free RulebaseHandler. Time to open it up.

  try {
    EngineHandlers[Handle].open(&RulebaseHandlers[RulebaseHandle]);                 // Try to open the handler.
  }                                                                                 // If an exception is thrown...
  catch(snf_EngineHandler::AllocationError& e)                                      // Catch and rethrow as appropriate.
    { EngineScan.unlock(); throw AllocationError(e.what()); }
  catch(snf_EngineHandler::Busy& e)
    { EngineScan.unlock(); throw Panic(e.what()); }                                 // Not busy above should not be busy now!!!
  catch(const std::exception& e) {
    EngineScan.unlock();
    throw e;
  }
  catch(...) {
    EngineScan.unlock();
    throw Panic("snf_MultiEngineHandler::OpenEngine() ???");
  }

  EngineScan.unlock();                                                              // If everything went well then UnLock
  return Handle;                                                                    // and return the happy new handle.
}

// snf_CloseEngine()
// Shut down this Engine handler.

void snf_MultiEngineHandler::CloseEngine(int EngineHandle){                         // Closing an engine handler.
  EngineScan.lock();                                                                // Serialize this, the object changes states.
  try {
  	EngineHandlers[EngineHandle].close();                                           // Try closing the handler.
  }
  catch(snf_EngineHandler::AllocationError& e)                                      // Catch and throw any exceptions as needed.
    { EngineScan.unlock(); throw AllocationError(e.what()); }
  catch(snf_EngineHandler::Busy& e)
    { EngineScan.unlock(); throw Busy(e.what()); }
  catch(const std::exception& e) {
      EngineScan.unlock();
      throw e;
  }
  catch(...) {
    EngineScan.unlock();
    throw Panic("snf_MultiEngineHandler::CloseEngine() ???");
  }
  EngineScan.unlock();                                                              // Unlock when we're closed.
}

// snf_Scan()
// Scan the MessageBuffer with this Engine.

int snf_MultiEngineHandler::Scan(int EngineHandle, const unsigned char* MessageBuffer, int MessageLength){

  // NOT serialized. Many scans at once, presumably one scan engine per thread.

  int ScanResult;                                                                   // ScanResult stays in scope.
  try {
  	ScanResult=EngineHandlers[EngineHandle]
  	  .scanMessage(MessageBuffer,MessageLength);                                    // Try the scan on the given engine.
  }
  catch(snf_EngineHandler::AllocationError& e) {                                    // Re-throw any exceptions as needed.
    throw AllocationError(e.what());
  }
  catch(snf_EngineHandler::Busy& e) { throw Busy(e.what()); }
  catch(const std::exception& e) { throw e; }
  catch(...) { throw Panic("snf_MultiEngineHandler::Scan() ???"); }
  return ScanResult;                                                                // Return the results.
}

// The Engine prvides detailed match results through this function.

int snf_MultiEngineHandler::getResults(int EngineHandle, snf_match* matchbfr){

  // NOT serialized. Many scans at once, presumably one scan engine per thread.

  int ResultCount;                                                                  // ResultCount stays in scope.

  try {
  	ResultCount=EngineHandlers[EngineHandle].getResults(matchbfr);                  // Try the scan on the given engine.
  }
  catch(snf_EngineHandler::AllocationError& e) {                                    // Re-throw any exceptions as needed.
    throw AllocationError(e.what());
  }
  catch(snf_EngineHandler::Busy& e) { throw Busy(e.what()); }
  catch(const std::exception& e) { throw e; }
  catch(...) { throw Panic("snf_MultiEngineHandler::getResults() ???"); }

  return ResultCount;                                                               // Return the results.
}

// The Engine provies the scan depth through this function.

int snf_MultiEngineHandler::getDepth(int EngineHandle){
  // NOT serialized. Many scans at once, presumably one scan engine per thread.

  int DepthResult;                                                                  // ScanResult stays in scope.

  try {
  	DepthResult=EngineHandlers[EngineHandle].getDepth();                            // Try the scan on the given engine.
  }
  catch(snf_EngineHandler::AllocationError& e) {                                    // Re-throw any exceptions as needed.
    throw AllocationError(e.what());
  }
  catch(snf_EngineHandler::Busy& e) { throw Busy(e.what()); }
  catch(const std::exception& e) { throw e; }
  catch(...) { throw Panic("snf_MultiEngineHandler::getDepth() ???"); }

  return DepthResult;                                                               // Return the results.
}