SNFMulti/SNFMulti.hpp

// SNFMulti.hpp
//
// (C) Copyright 2006 - 2009 ARM Research Labs, LLC.
// See www.armresearch.com for the copyright terms.
//
// 20060121_M
// This file creates an API for multi-threaded systems to use the SNF engine.
//
// This API is C++ oriented, meaning it throws exceptions and so forth.
// For use in shared objects and DLLs, the functions in here will be wrapped
// in a C style interface appropriate to that platform.
//
// The interface is based on the following structure.
//
// The application "Opens" one or more rulebases.
// The application "Opens" some number of scanners referencing opened rulebases.
// Each scanner handles one thread's worth of scanning, so it is presumed that
// each processing thread in the calling application will have one scanner to itself.
//
// Rulebases can be reloaded asynchronously. The scanner's grab a reference to the
// rulebase each time they restart. The grabbing and swapping in of new rulebases is
// a very short critical section.

#ifndef _ARM_SNFMulti
#define _ARM_SNFMulti

#include <stdexcept>
#include <sys/types.h>
#include <sys/stat.h>
#include <ctime>
#include <string>
#include "../CodeDweller/faults.hpp"
#include "../CodeDweller/threading.hpp"
#include "GBUdb.hpp"
#include "FilterChain.hpp"
#include "snf_engine.hpp"
#include "snf_match.h"
#include "snfCFGmgr.hpp"
#include "snfLOGmgr.hpp"
#include "snfNETmgr.hpp"
#include "snfGBUdbmgr.hpp"
#include "snfXCImgr.hpp"
#include "snf_saccades.hpp"

#include <cassert>

extern const char* SNF_ENGINE_VERSION;

// snf Result Code Constants

const int snf_SUCCESS = 0;
const int snf_ERROR_CMD_LINE = 65;
const int snf_ERROR_LOG_FILE = 66;
const int snf_ERROR_RULE_FILE = 67;
const int snf_ERROR_RULE_DATA = 68;
const int snf_ERROR_RULE_AUTH = 73;
const int snf_ERROR_MSG_FILE = 69;
const int snf_ERROR_ALLOCATION = 70;
const int snf_ERROR_BAD_MATRIX = 71;
const int snf_ERROR_MAX_EVALS = 72;
const int snf_ERROR_UNKNOWN = 99;

// Settings & Other Constants

const int snf_ScanHorizon = 32768;                                              // Maximum length of message to check.
const int snf_MAX_RULEBASES = 10;                                               // 10 Rulebases is plenty. Most use just 1
const int snf_MAX_SCANNERS = 500;                                               // 500 Scanners at once should be plenty

const int SHUTDOWN = -999;                                                      // Shutdown Cursor Value.

// snfCFGPacket encapsulates configuration and rulebase data.
// The rulebase handler can write to it.
// Others can only read from it.
// The engine handler creates and owns one of these. It uses it to
// grab() and drop() cfg and rulebase data from the rulebase handler.

class snf_RulebaseHandler;                                                      // We need to know this exists.

class snfCFGPacket {                                                            // Our little bundle of, er, cfg stuff.

  friend class snf_RulebaseHandler;                                             // RulebaseHandler has write access.

  private:
    snf_RulebaseHandler* MyRulebase;                                            // Where to grab() and drop()
    TokenMatrix* MyTokenMatrix;                                                 // We combine the current token matrix
    snfCFGData* MyCFGData;                                                      // and the current cfg data for each scan.

    set<int> RulePanics;                                                        // Set of known rule panic IDs.

  public:
    snfCFGPacket(snf_RulebaseHandler* R);                                       // Constructor grab()s the Rulebase.
    ~snfCFGPacket();                                                            // Destructor drop()s the Rulebase.

    TokenMatrix* Tokens();                                                      // Consumers read the Token Matrix and
    snfCFGData* Config();                                                       // the snfCFGData.

    bool bad();                                                                 // If anything is missing it's not good.

    bool isRulePanic(int R);                                                    // Test for a rule panic.
};

class ScriptCaller : private Thread {                                           // Calls system() in separate thread.
  private:
    Mutex MyMutex;                                                              // Protects internal data.
    string SystemCallText;                                                      // Text to send to system().
    Timeout GuardTimer;                                                         // Guard time between triggers.
    volatile bool GoFlag;                                                       // Go flag true when triggered.
    volatile bool DieFlag;                                                      // Die flag when it's time to leave.

    string ScriptToRun();                                                       // Safely grab the script.
    bool hasGuardExpired();                                                     // True if guard time has expired.
    void myTask();                                                              // Thread task overload.

    volatile int myLastResult;                                                  // Last result of system() call.

  public:
    ScriptCaller(string Name);                                                  // Constructor.
    ~ScriptCaller();                                                            // Destructor.

    void SystemCall(string S);                                                  // Set system call text.
    void GuardTime(int T);                                                      // Change guard time.
    void trigger();                                                             // Trigger if possible.
    int LastResult();                                                           // Return myLastResult.

    const static ThreadType Type;                                               // The thread's type.

    const static ThreadState CallingSystem;                                     // State when in system() call.
    const static ThreadState PendingGuardTime;                                  // State when waiting for guard time.
    const static ThreadState StandingBy;                                        // State when waiting around.
    const static ThreadState Disabled;                                          // State when unable to run.
};

class snf_Reloader : private Thread {                                           // Rulebase maintenance thread.
  private:

    snf_RulebaseHandler& MyRulebase;                                            // We know our rulebase.
    bool TimeToStop;                                                            // We know if it's time to stop.

    string RulebaseFileCheckName;                                               // We keep track of these files.
    string ConfigFileCheckName;
    string IgnoreListCheckFileName;
    time_t RulebaseFileTimestamp;                                               // We watch their timestamps.
    time_t ConfigurationTimestamp;
    time_t IgnoreListTimestamp;

    void captureFileStats();                                                    // Get stats for later comparison.
    bool StatsAreDifferent();                                                   // Check file stats for changes.

    void myTask();                                                              // How do we do this refresh thing?

    ScriptCaller RulebaseGetter;                                                // Reloader owns a RulebaseGetter.
    bool RulebaseGetterIsTurnedOn;                                              // True if we should run the getter.
    void captureGetterConfig();                                                 // Get RulebaseGetter config.

  public:
    snf_Reloader(snf_RulebaseHandler& R);                                       // Setup takes some work.
    ~snf_Reloader();                                                            // Tear down takes some work.

    const static ThreadType Type;                                               // The thread's type.

};

class snf_RulebaseHandler {                                                     // Engine Core Manager.

  friend class snfCFGPacket;

  private:

    Mutex MyMutex;                                                              // This handler's mutex.

    snf_Reloader* MyReloader;                                                   // Reloader engine (when in use).

    int volatile ReferenceCount;                                                // Associated scanners count.

    snfCFGData* volatile Configuration;                                         // Configuration for this handler.
	TokenMatrix* volatile Rulebase;                                             // Rulebase for this handler.
	int volatile CurrentCount;                                                  // Active current scanners count.

	TokenMatrix* volatile OldRulebase;                                          // Retiring rulebase holder.
	int volatile RetiringCount;                                                 // Active retiring scanners count.

	bool volatile RefreshInProgress;                                            // Flag for locking the refresh process.

	int volatile MyGeneration;                                                  // Generation (reload) number.

	void _snf_LoadNewRulebase();                                                // Internal function to load new rulebase.

    Mutex XCIServerCommandMutex;                                                // XCI Server Command Serializer.
    snfXCIServerCommandHandler* myXCIServerCommandHandler;                      // ptr to Installed Srv Cmd Handler.

    void grab(snfCFGPacket& CP);                                                // Activate this Rulebase for a scan.
    void drop(snfCFGPacket& CP);                                                // Deactiveate this Rulebase after it.

  public:

    class ConfigurationError : public runtime_error {                           // When the configuration won't load.
      public: ConfigurationError(const string& w):runtime_error(w) {}
    };
    class FileError : public runtime_error {                                    // Exception: rulebase file won't load.
      public: FileError(const string& w):runtime_error(w) {}
    };
    class AuthenticationError : public runtime_error {                          // Exception when authentication fails.
      public: AuthenticationError(const string& w):runtime_error(w) {}
    };
    class IgnoreListError : public runtime_error {                              // When the ignore list won't load.
      public: IgnoreListError(const string& w):runtime_error(w) {}
    };
    class AllocationError : public runtime_error {                              // Exception when we can't allocate something.
      public: AllocationError(const string& w):runtime_error(w) {}
    };
    class Busy : public runtime_error {                                         // Exception when there is a collision.
      public: Busy(const string& w):runtime_error(w) {}
    };
    class Panic : public runtime_error {                                        // Exception when something else happens.
      public: Panic(const string& w):runtime_error(w) {}
    };

    //// Plugin Components.

    snfCFGmgr MyCFGmgr;                                                         // Configuration manager.
    snfLOGmgr MyLOGmgr;                                                         // Logging manager.
    snfNETmgr MyNETmgr;                                                         // Communications manager.
    snfGBUdbmgr MyGBUdbmgr;                                                     // GBUdb manager.
    GBUdb MyGBUdb;                                                              // GBUdb for this rulebase.
    snfXCImgr MyXCImgr;                                                         // XCI manager.

    //// Methods.

    snf_RulebaseHandler():                                                      // Initialization is straight forward.
      MyReloader(0),
      ReferenceCount(0),
      Rulebase(NULL),
      CurrentCount(0),
      OldRulebase(NULL),
      RetiringCount(0),
      RefreshInProgress(false),
      MyGeneration(0),
      myXCIServerCommandHandler(0){
        MyNETmgr.linkLOGmgr(MyLOGmgr);                                          // Link the NET manager to the LOGmgr.
        MyNETmgr.linkGBUdbmgr(MyGBUdbmgr);                                      // Link the NET manager to the GBUdbmgr.
        MyGBUdbmgr.linkGBUdb(MyGBUdb);                                          // Link the GBUdb manager to it's db.
        MyGBUdbmgr.linkLOGmgr(MyLOGmgr);                                        // Link the GBUdb manager to the LOGmgr.
        MyLOGmgr.linkNETmgr(MyNETmgr);                                          // Link the LOG manager to the NETmgr.
        MyLOGmgr.linkGBUdb(MyGBUdb);                                            // Link the LOG manager to the GBUdb.
        MyXCImgr.linkHome(this);                                                // Link the XCI manager to this.
    }

    ~snf_RulebaseHandler();                                                     // Shutdown checks for safety.

    bool isReady();                                                             // Is the object is active.
    bool isBusy();                                                              // Is a refresh/open in progress.
    int getReferenceCount();                                                    // How many Engines using this handler.
    int getCurrentCount();                                                      // How many Engines active in the current rb.
    int getRetiringCount();                                                     // How many Engines active in the old rb.
    void open(const char* path,                                                 // Lights up this hanlder.
              const char* licenseid,
              const char* authentication);

    bool AutoRefresh(bool On);                                                  // Turn on/off auto refresh.
    bool AutoRefresh();                                                         // True if AutoRefresh is on.
    void refresh();                                                             // Reloads the rulebase and config.

    void close();                                                               // Closes this handler.

    void use();                                                                 // Make use of this Rulebase Handler.
    void unuse();                                                               // Finish with this Rulebase Handler.

    int Generation();                                                           // Returns the generation number.

    void addRulePanic(int RuleID);                                              // Synchronously add a RulePanic.
    bool testXHDRInjectOn();                                                    // Safely look ahead at XHDRInjectOn.

    IPTestRecord& performIPTest(IPTestRecord& I);                               // Perform an IP test.
    void logThisIPTest(IPTestRecord& I, string Action);                         // Log an IP test result & action.

    void logThisError(string ContextName, int Code, string Text);               // Log an error message.
    void logThisInfo(string ContextName, int Code, string Text);                // Log an informational message.
    string PlatformVersion(string NewPlatformVersion);                          // Set platform version info.
    string PlatformVersion();                                                   // Get platform version info.
    string PlatformConfiguration();                                             // Get platform configuration.
    string EngineVersion();                                                     // Get engine version info.

    void XCIServerCommandHandler(snfXCIServerCommandHandler& XCH);              // Registers a new XCI Srvr Cmd handler.
    string processXCIServerCommandRequest(snf_xci& X);                          // Handle a parsed XCI Srvr Cmd request.
};

// IPTestEngine w/ GBUdb interface.
// This will plug into the FilterChain to evaluate IPs on the fly.

class snf_IPTestEngine : public FilterChainIPTester {

  private:

    GBUdb* Lookup;                                                              // Where we find our GBUdb.
    snfScanData* ScanData;                                                      // Where we find our ScanData.
    snfCFGData* CFGData;                                                        // Where we find our CFG data.
    snfLOGmgr* LOGmgr;                                                          // Where we find our LOG manager.

  public:

    snf_IPTestEngine();                                                         // Initialize internal pointers to NULL.
    void setGBUdb(GBUdb& G);                                                    // Setup the GBUdb lookup.
    void setScanData(snfScanData& D);                                           // Setup the ScanData object.
    void setCFGData(snfCFGData& C);                                             // (Re)Set the config data to use.
    void setLOGmgr(snfLOGmgr& L);                                               // Setup the LOGmgr to use.

    string& test(string& input, string& output);                                // Our obligatory test function.
};


class snf_SaccadesHandler {
    private:
        Mutex MyMutex;
        saccades_engine MyEngine;

        void lockAndLearn(vector<saccade>& Matches) {
            ScopeMutex SafetyFirst(MyMutex);
            MyEngine.learn(Matches);
        }

        vector<saccade> grabSaccades() {
            ScopeMutex SafetyFirst(MyMutex);
            return MyEngine.recall();
        }

        int TimeToPeekCounter;
        static const int TimeToPeekReset = 32;

    public:

        static const int AlwaysScanLength = 2048;

        snf_SaccadesHandler() :
            MyEngine(128),
            TimeToPeekCounter(0) {}

        void applySaccades(EvaluationMatrix* Scanner, vector<unsigned char>& Data);
        void learnMatches(MatchRecord* Matches);
};

// How to spot strangers in the IP reputations.

class snf_IPStrangerList {
  private:
    Mutex myMutex;
    set<IP4Address> listA;
    set<IP4Address> listB;
    bool usingANotB;
    Timeout listExpiration;

    set<IP4Address>& myActiveList() {
        if(usingANotB) return listA;
        else return listB;
    }

    void swapOutOldLists() {
        if(listExpiration.isExpired()) {
            usingANotB = !usingANotB;
            myActiveList().clear();
            listExpiration.restart();
        }
    }

    set<IP4Address>& myCurrentList() {
        swapOutOldLists();
        return myActiveList();
    }

  public:
    static const int TwoHours = (2 * (60 * (60 * 1000)));
    snf_IPStrangerList() : usingANotB(true), listExpiration(TwoHours) {}

    bool isStranger(IP4Address a) {
        ScopeMutex JustMe(myMutex);
        swapOutOldLists();
        bool foundStranger = (0 < listA.count(a)) || (0 < listB.count(a));
        return foundStranger;
    }

    void addStranger(IP4Address a) {
        ScopeMutex JustMe(myMutex);
        myCurrentList().insert(a);
    }
};

// Here's where we pull it all together.

class snf_EngineHandler {

	private:

      Mutex MyMutex;                                                            // This handler's mutex.
      Mutex FileScan;                                                           // File scan entry mutex.

	  EvaluationMatrix* volatile CurrentMatrix;                                 // Matrix for the latest scan.
	  snf_RulebaseHandler* volatile MyRulebase;                                 // My RulebaseHandler.

	  snfScanData MyScanData;                                                   // Local snfScanData record.
	  snf_IPTestEngine MyIPTestEngine;                                          // Local IP Test Engine.

      int ResultsCount;                                                         // Count of Match Records for getResults
      int ResultsRemaining;                                                     // Count of Match Records ahead of cursor.
      MatchRecord* FinalResult;                                                 // Final (winning) result of the scan.
	  MatchRecord* ResultCursor;                                                // Current Match Record for getResults.

      string extractMessageID(const unsigned char* Msg, const int Len);         // Get log safe Message-ID or substitute.

    public:

      class FileError : public runtime_error {                                  // Exception when a file won't open.
        public: FileError(const string& w):runtime_error(w) {}
      };
      class XHDRError : public runtime_error {                                  // Exception when XHDR Inject/File fails.
        public: XHDRError(const string& w):runtime_error(w) {}
      };
      class BadMatrix : public runtime_error {                                  // Exception out of bounds of matrix.
        public: BadMatrix(const string& w):runtime_error(w) {}
      };
      class MaxEvals : public runtime_error {                                   // Exception too many evaluators.
        public: MaxEvals(const string& w):runtime_error(w) {}
      };
      class AllocationError : public runtime_error {                            // Exception when we can't allocate something.
        public: AllocationError(const string& w):runtime_error(w) {}
      };
      class Busy : public runtime_error {                                       // Exception when there is a collision.
        public: Busy(const string& w):runtime_error(w) {}
      };
      class Panic : public runtime_error {                                      // Exception when something else happens.
        public: Panic(const string& w):runtime_error(w) {}
      };

      snf_EngineHandler():                                                      // Initialization is simple.
        CurrentMatrix(NULL),
        MyRulebase(NULL),
        MyScanData(snf_ScanHorizon),
        ResultsCount(0),
        ResultsRemaining(0),
        ResultCursor(NULL) {}

      ~snf_EngineHandler();                                                     // Shutdown clenas up and checks for safety.

      void open(snf_RulebaseHandler* Handler);                                  // Light up the engine.
      bool isReady();                                                           // Is the Engine good to go? (doubles as busy)
      void close();                                                             // Close down the engine.

      int scanMessageFile(                                                      // Scan this message file.
        const string MessageFilePath,                                           // -- this is the file (and id)
        const int MessageSetupTime = 0,                                         // -- setup time already used.
        const IP4Address MessageSource = 0UL                                    // -- message source IP (for injection).
      );

	  int scanMessage(                                                          // Scan this message.
	    const unsigned char* MessageBuffer,                                     // -- this is the message buffer.
	    const int MessageLength,                                                // -- this is the length of the buffer.
        const string MessageName = "",                                          // -- this is the message identifier.
        const int MessageSetupTime = 0,                                         // -- setup time used (for logging).
        const IP4Address MessageSource = 0UL                                    // -- message source IP (for injection).
      );

	  int getResults(snf_match* MatchBuffer);                                   // Get the next match buffer.
	  int getDepth();                                                           // Get the scan depth.

	  const string getClassicLog();                                             // Get classic log entries for last scan.
	  const string getXMLLog();                                                 // Get XML log entries or last scan.
	  const string getXHDRs();                                                  // Get XHDRs for last scan.
};

// Here's the class that pulls it all together.

class snf_MultiEngineHandler {

  private:

    Mutex RulebaseScan;                                                         // This handler's mutex.
    int RulebaseCursor;                                                         // Next Rulebase to search.
    snf_RulebaseHandler RulebaseHandlers[snf_MAX_RULEBASES];                    // Array of Rulebase Handlers

    int RoundRulebaseCursor();                                                  // Gets round robin Rulebase handle candidates.

    Mutex EngineScan;                                                           // Serializes searching the Engine list.
    int EngineCursor;                                                           // Next Engine to search.
    snf_EngineHandler EngineHandlers[snf_MAX_SCANNERS];                         // Array of Engine Handlers

    int RoundEngineCursor();                                                    // Gets round robin Engine handle candidates.

  public:

    class TooMany : public runtime_error {                                      // Exception when no more handle slots.
      public: TooMany(const string& w):runtime_error(w) {}
    };
    class FileError : public runtime_error {                                    // Exception when a file won't open.
      public: FileError(const string& w):runtime_error(w) {}
    };
    class AuthenticationError : public runtime_error {                          // Exception when authentication fails.
      public: AuthenticationError(const string& w):runtime_error(w) {}
    };
    class AllocationError : public runtime_error {                              // Exception when we can't allocate something.
      public: AllocationError(const string& w):runtime_error(w) {}
    };
    class Busy : public runtime_error {                                         // Exception when there is a collision.
      public: Busy(const string& w):runtime_error(w) {}
    };
    class Panic : public runtime_error {                                        // Exception when something else happens.
      public: Panic(const string& w):runtime_error(w) {}
    };

    snf_MultiEngineHandler():
      RulebaseCursor(0),
      EngineCursor(0) {}

    ~snf_MultiEngineHandler();                                                  // Clean up, safety check, shut down.

    // snf_OpenRulebase()
    // Grab the first available rulebse handler and light it up.

    int OpenRulebase(const char* path, const char* licenseid, const char* authentication);

    // snf_RefreshRulebase()
    // Reload the rulebase associated with the handler.

    void RefreshRulebase(int RulebaseHandle);

    // snf_CloseRulebase()
    // Shut down this Rulebase handler.

    void CloseRulebase(int RulebaseHandle);

    // snf_OpenEngine()
    // Grab the first available Engine handler and light it up

    int OpenEngine(int RulebaseHandle);

    // snf_CloseEngine()
    // Shut down this Engine handler.

    void CloseEngine(int EngineHandle);

    // snf_Scan()
    // Scan the MessageBuffer with this Engine.

    int Scan(int EngineHandle, const unsigned char* MessageBuffer, int MessageLength);

    // The Engine prvides detailed match results through this function.

    int getResults(int EngineHandle, snf_match* matchbfr);

    // The Engine provies the scan depth through this function.

    int getDepth(int EngineHandle);

};

#endif