Implemented AI based "saccades" engine to improve scanning efficiency. The Saccades engine borrows from vision systems research and allows SNF to learn important message structures and adapt it's scanning technique in real-time. Using this engine SNF can often avoid scanning large portions of each message without missing important content resulting in potentially 10 fold improvements in scanner efficiency.

git-svn-id: https://svn.microneil.com/svn/SNFMulti/trunk@53 dc71a809-1921-45c4-985c-09c81d0142d9

SNFMulti.cpp

@@ -14,7 +14,9 @@
 #include <cstdlib>
 
 #include <sstream>
-#include "SNFMulti.hpp"
+#include "SNFMulti.hpp"
+#include "snf_saccades.hpp"
+
 #include "../CodeDweller/timing.hpp"
 
 //#include "../nvwa-0.6/nvwa/debug_new.h"
@@ -23,7 +25,7 @@ using namespace std;
 
 //// Version Info
 
-const char* SNF_ENGINE_VERSION = "SNFMulti Engine Version 3.0.24 Build: " __DATE__ " " __TIME__;
+const char* SNF_ENGINE_VERSION = "SNFMulti Engine Version 3.1.0 Build: " __DATE__ " " __TIME__;
 
 //// Script Caller Methods
 
@@ -1322,7 +1324,56 @@ void captureMatchRecord(snf_match& M, MatchRecord* R) {
   M.index = R->MatchStartPosition;
   M.endex = R->MatchEndPosition;
 }
-
+
+void snf_SaccadesHandler::applySaccades(EvaluationMatrix* Scanner, vector<unsigned char>& Data) {
+    if(NULL == Scanner) return;
+
+    bool isTimeToPeek = (0 >= TimeToPeekCounter);
+    if(isTimeToPeek) {
+        TimeToPeekCounter = TimeToPeekReset;
+        return;
+    } else {
+        --TimeToPeekCounter;
+    }
+
+    vector<saccade> Saccades = grabSaccades();
+    for(vector<saccade>::iterator i = Saccades.begin(); i != Saccades.end(); i++) {
+        const saccade& s = (*i);
+        if(s.start >= Data.size()) break;
+        Scanner->evaluateSegment(Data, s.start, s.finish);
+    }
+}
+
+bool isLearnableMatch(MatchRecord* m) {
+    bool isGoodSymbol = (0 <= m->RuleGroup() && 64 > m->RuleGroup());
+    bool isBeyondAlwaysScan = (snf_SaccadesHandler::AlwaysScanLength < m->MatchEndPosition);
+    return (isGoodSymbol && isBeyondAlwaysScan);
+}
+
+void snf_SaccadesHandler::learnMatches(MatchRecord* Matches) {
+    if(NULL == Matches) return;
+
+    vector<saccade> MatchesToLearn;
+    saccade WatchForHeaderWhiteRules(0, AlwaysScanLength);
+    MatchesToLearn.push_back(WatchForHeaderWhiteRules);
+
+    for(MatchRecord* m = Matches; NULL != m; m = m->NextMatchRecord) {
+        if(isLearnableMatch(m)) {
+            MatchesToLearn.push_back(
+              saccade(
+                m->MatchStartPosition,
+                m->MatchEndPosition)
+            );
+        }
+    }
+
+    if(0 < MatchesToLearn.size()) {
+        lockAndLearn(MatchesToLearn);
+    }
+}
+
+static snf_SaccadesHandler SaccadeBrain;
+
 int snf_EngineHandler::scanMessage(                                             // Scan this message (in buffer).
   const unsigned char* inputMessageBuffer,                                      // -- this is the message buffer.
   const int inputMessageLength,                                                 // -- this is the length of the buffer.
@@ -1545,8 +1596,17 @@ int snf_EngineHandler::scanMessage(
       DebugInfo = "scanMessage() EvaluateThis(FilteredData)";                   // If we panic, here we are.
 
       if(false == MyScanData.GBUdbTruncateExecuted) {                           // If we haven't already truncated:
-          for(int a = 0, b = MyScanData.FilteredData.size(); a < b; a++)        // Scan through the filtered data one
-            CurrentMatrix->EvaluateThis(MyScanData.FilteredData[a]);            // byte at a time.
+          //for(int a = 0, b = MyScanData.FilteredData.size(); a < b; a++)        // Scan through the filtered data one
+          //  CurrentMatrix->EvaluateThis(MyScanData.FilteredData[a]);            // byte at a time.
+
+          unsigned int fullLength = MyScanData.FilteredData.size();
+
+          SaccadeBrain.applySaccades(CurrentMatrix, MyScanData.FilteredData);
+          bool messageNotRecognized = (NULL == CurrentMatrix->ResultList);
+          if(messageNotRecognized) {
+              CurrentMatrix->evaluateSegment(MyScanData.FilteredData, 0, fullLength);
+              SaccadeBrain.learnMatches(CurrentMatrix->ResultList);
+          }
       }
 
       DebugInfo = "scanMessage() Scan Data Complete";                           // If we panic, here we are.
@@ -1628,7 +1688,7 @@ int snf_EngineHandler::scanMessage(
   int S = NO_SYMBOL;                                                            // so we start there and work down.
 
   snf_match TmpSNFMatch;                                                        // We'll need a buffer for our matches.
-
+
   while(NULL!=ResultCursor) {                                                   // While we have records to process...
     captureMatchRecord(TmpSNFMatch, ResultCursor);                              // grab the next record and evaluate it.
 
@@ -1659,7 +1719,7 @@ int snf_EngineHandler::scanMessage(
     ResultsCount++;
     ResultCursor=ResultCursor->NextMatchRecord;
   }
-
+
   if(NO_SYMBOL != S) {                                                          // If a pattern match was detected then
       MyScanData.PatternWasFound = true;                                        // trip the flag and record the
       MyScanData.PatternID = FinalResult->RuleId();                             // Rule ID and the

SNFMulti.hpp

@@ -39,8 +39,9 @@
 #include "snfLOGmgr.hpp"
 #include "snfNETmgr.hpp"
 #include "snfGBUdbmgr.hpp"
-#include "snfXCImgr.hpp"
-
+#include "snfXCImgr.hpp"
+#include "snf_saccades.hpp"
+
 #include <cassert>
 
 extern const char* SNF_ENGINE_VERSION;
@@ -188,8 +189,8 @@ class snf_RulebaseHandler {
     snfXCIServerCommandHandler* myXCIServerCommandHandler;                      // ptr to Installed Srv Cmd Handler.
 
     void grab(snfCFGPacket& CP);                                                // Activate this Rulebase for a scan.
-    void drop(snfCFGPacket& CP);                                                // Deactiveate this Rulebase after it.
-
+    void drop(snfCFGPacket& CP);                                                // Deactiveate this Rulebase after it.
+
   public:
 
     class ConfigurationError : public runtime_error {                           // When the configuration won't load.
@@ -234,7 +235,7 @@ class snf_RulebaseHandler {
       RetiringCount(0),
       RefreshInProgress(false),
       MyGeneration(0),
-      myXCIServerCommandHandler(0) {
+      myXCIServerCommandHandler(0){
         MyNETmgr.linkLOGmgr(MyLOGmgr);                                          // Link the NET manager to the LOGmgr.
         MyNETmgr.linkGBUdbmgr(MyGBUdbmgr);                                      // Link the NET manager to the GBUdbmgr.
         MyGBUdbmgr.linkGBUdb(MyGBUdb);                                          // Link the GBUdb manager to it's db.
@@ -280,7 +281,7 @@ class snf_RulebaseHandler {
     string EngineVersion();                                                     // Get engine version info.
 
     void XCIServerCommandHandler(snfXCIServerCommandHandler& XCH);              // Registers a new XCI Srvr Cmd handler.
-    string processXCIServerCommandRequest(snf_xci& X);                          // Handle a parsed XCI Srvr Cmd request.
+    string processXCIServerCommandRequest(snf_xci& X);                          // Handle a parsed XCI Srvr Cmd request.
 };
 
 // IPTestEngine w/ GBUdb interface.
@@ -305,6 +306,37 @@ class snf_IPTestEngine : public FilterChainIPTester {
 
     string& test(string& input, string& output);                                // Our obligatory test function.
 };
+
+
+class snf_SaccadesHandler {
+    private:
+        Mutex MyMutex;
+        saccades_engine MyEngine;
+
+        void lockAndLearn(vector<saccade>& Matches) {
+            ScopeMutex SafetyFirst(MyMutex);
+            MyEngine.learn(Matches);
+        }
+
+        vector<saccade> grabSaccades() {
+            ScopeMutex SafetyFirst(MyMutex);
+            return MyEngine.recall();
+        }
+
+        int TimeToPeekCounter;
+        static const int TimeToPeekReset = 32;
+
+    public:
+
+        static const int AlwaysScanLength = 2048;
+
+        snf_SaccadesHandler() :
+            MyEngine(128),
+            TimeToPeekCounter(0) {}
+
+        void applySaccades(EvaluationMatrix* Scanner, vector<unsigned char>& Data);
+        void learnMatches(MatchRecord* Matches);
+};
 
 // Here's where we pull it all together.
 

snf_engine.cpp

@@ -33,7 +33,8 @@
 #include <cstdlib>
 #include <fstream>
 #include <iostream>
-#include <string>
+#include <string>
+#include <vector>
 #include "../CodeDweller/mangler.hpp"
 #include "snf_engine.hpp"
 
@@ -662,7 +663,31 @@ void EvaluationMatrix::DropEvaluator() { // Drops the current evaluator from the
   --CountOfEvaluators;                                                          // Reduce our evaluator count.
 
 }
-
+
+Evaluator* findEvaluatorListTail(Evaluator* head) {
+    Evaluator* next = head;
+    while(NULL != (next->NextEvaluator)) next = next->NextEvaluator;
+    return next;
+}
+
+void EvaluationMatrix::dropAllEvaluators() {
+    bool haveActiveEvaluators = (NULL != EvaluatorList);
+    if(haveActiveEvaluators) {
+        Evaluator* tail = findEvaluatorListTail(EvaluatorList);
+        tail->NextEvaluator = EvaluatorCache;
+        EvaluatorCache = EvaluatorList;
+    }
+    PreviousEvaluator = NULL;
+    CurrentEvaluator = NULL;
+    EvaluatorList = NULL;
+    CountOfEvaluators = 0;
+}
+
+void EvaluationMatrix::restartEngineAt(int newCharacterCount) {
+    dropAllEvaluators();
+    CountOfCharacters = newCharacterCount;
+}
+
 
 // EvaluationMatrix::EvaluateThis()
 //
@@ -763,3 +788,9 @@ int EvaluationMatrix::EvaluateThis(unsigned short int i) {
 
   return PassResult;  // When we're finished, return the last known result.
 }
+
+void EvaluationMatrix::evaluateSegment(vector<unsigned char>& data, unsigned int start, unsigned int finish) {
+    restartEngineAt(start);
+    finish = (finish < data.size()) ? finish : data.size();
+    for(unsigned int a = start; a < finish; a++) EvaluateThis(data[a]);
+}

snf_engine.hpp

@@ -38,7 +38,8 @@
 #include <cstdlib>
 #include <fstream>
 #include <iostream>
-#include <string>
+#include <string>
+#include <vector>
 #include <exception>
 #include "../CodeDweller/faults.hpp"
 #include "../CodeDweller/mangler.hpp"
@@ -408,7 +409,9 @@ class EvaluationMatrix {
     // evaluator node if it exists. Therefore, the caller should skip it's normal
     // list itteration code when this function has been called.
 
-    void DropEvaluator();
+    void DropEvaluator();
+
+    void dropAllEvaluators();
 
   public:
 
@@ -474,7 +477,11 @@ class EvaluationMatrix {
     // EvaluateThis() Moves each evaluator with the current character and creates a new
     // evaluator for the current spot in the input file to make all rules global.
 
-    int EvaluateThis(unsigned short int i);
+    int EvaluateThis(unsigned short int i);
+
+    void evaluateSegment(vector<unsigned char>& data, unsigned int start, unsigned int finish);
+
+    void restartEngineAt(int newCharacterCount);
 
     EvaluationMatrix(TokenMatrix* m) { // Constructor w/ pointer to Token Matrix...