Engine 3.2.2 Removed saccades

SNFMulti.cpp

 
 #include <sstream>
 #include "SNFMulti.hpp"
-#include "snf_saccades.hpp"
 
 #include "../CodeDweller/timing.hpp"
 
 
 //// Version Info
 
-const char* SNF_ENGINE_VERSION = "SNFMulti Engine Version 3.2.1 Build: " __DATE__ " " __TIME__;
+const char* SNF_ENGINE_VERSION = "SNFMulti Engine Version 3.2.2 Build: " __DATE__ " " __TIME__;
 
 //// Script Caller Methods
 
   M.endex = R->MatchEndPosition;
 }
 
-void snf_SaccadesHandler::applySaccades(EvaluationMatrix* Scanner, std::vector<unsigned char>& Data) {
-    if(NULL == Scanner) return;
-
-    bool isTimeToPeek = (0 >= TimeToPeekCounter);
-    if(isTimeToPeek) {
-        TimeToPeekCounter = TimeToPeekReset;
-        return;
-    } else {
-        --TimeToPeekCounter;
-    }
-
-    std::vector<saccade> Saccades = grabSaccades();
-    for(std::vector<saccade>::iterator i = Saccades.begin(); i != Saccades.end(); i++) {
-        const saccade& s = (*i);
-        if(s.start >= Data.size()) break;
-        Scanner->evaluateSegment(Data, s.start, s.finish);
-    }
-}
-
-bool isLearnableMatch(MatchRecord* m) {
-    bool isGoodSymbol = (0 <= m->RuleGroup() && 64 > m->RuleGroup());
-    bool isBeyondAlwaysScan = (snf_SaccadesHandler::AlwaysScanLength < m->MatchEndPosition);
-    return (isGoodSymbol && isBeyondAlwaysScan);
-}
-
-void snf_SaccadesHandler::learnMatches(MatchRecord* Matches) {
-    if(NULL == Matches) return;
-
-    std::vector<saccade> MatchesToLearn;
-    saccade WatchForHeaderWhiteRules(0, AlwaysScanLength);
-    MatchesToLearn.push_back(WatchForHeaderWhiteRules);
-
-    for(MatchRecord* m = Matches; NULL != m; m = m->NextMatchRecord) {
-        if(isLearnableMatch(m)) {
-            MatchesToLearn.push_back(
-              saccade(
-                m->MatchStartPosition,
-                m->MatchEndPosition)
-            );
-        }
-    }
-
-    if(0 < MatchesToLearn.size()) {
-        lockAndLearn(MatchesToLearn);
-    }
-}
-
-static snf_SaccadesHandler SaccadeBrain;
 static snf_IPStrangerList StrangersList;
 
 int snf_EngineHandler::scanMessage(                                             // Scan this message (in buffer).
       DebugInfo = "scanMessage() EvaluateThis(FilteredData)";                   // If we panic, here we are.
 
       if(false == MyScanData.GBUdbTruncateExecuted) {                           // If we haven't already truncated:
-          //for(int a = 0, b = MyScanData.FilteredData.size(); a < b; a++)        // Scan through the filtered data one
-          //  CurrentMatrix->EvaluateThis(MyScanData.FilteredData[a]);            // byte at a time.
-
-          /** 20200618_M Experiment stripping out saccades to check performance hit
-          unsigned int fullLength = MyScanData.FilteredData.size();
-
-          SaccadeBrain.applySaccades(CurrentMatrix, MyScanData.FilteredData);
-          bool messageNotRecognized = (NULL == CurrentMatrix->ResultList);
-          if(messageNotRecognized) {
-              CurrentMatrix->evaluateSegment(MyScanData.FilteredData, 0, fullLength);
-              SaccadeBrain.learnMatches(CurrentMatrix->ResultList);
-          }
-          **/
         size_t fullLength = MyScanData.FilteredData.size();
-        CurrentMatrix->evaluateSegment(MyScanData.FilteredData, 0, fullLength); // Only do a full scan -- see comment above
+        CurrentMatrix->evaluateSegment(MyScanData.FilteredData, 0, fullLength); // Scan all the things!
       }
 
       DebugInfo = "scanMessage() Scan Data Complete";                           // If we panic, here we are.

SNFMulti.hpp

 #include "snfNETmgr.hpp"
 #include "snfGBUdbmgr.hpp"
 #include "snfXCImgr.hpp"
-#include "snf_saccades.hpp"
 
 #include <cassert>
 
     std::string& test(std::string& input, std::string& output);                 // Our obligatory test function.
 };
 
-
-class snf_SaccadesHandler {
-    private:
-        cd::Mutex MyMutex;
-        saccades_engine MyEngine;
-
-        void lockAndLearn(std::vector<saccade>& Matches) {
-            cd::ScopeMutex SafetyFirst(MyMutex);
-            MyEngine.learn(Matches);
-        }
-
-        std::vector<saccade> grabSaccades() {
-            cd::ScopeMutex SafetyFirst(MyMutex);
-            return MyEngine.recall();
-        }
-
-        int TimeToPeekCounter;
-        static const int TimeToPeekReset = 32;
-
-    public:
-
-        static const int AlwaysScanLength = 2048;
-
-        snf_SaccadesHandler() :
-            MyEngine(128),
-            TimeToPeekCounter(0) {}
-
-        void applySaccades(EvaluationMatrix* Scanner, std::vector<unsigned char>& Data);
-        void learnMatches(MatchRecord* Matches);
-};
-
 // How to spot strangers in the IP reputations.
 
 class snf_IPStrangerList {

scanner.cpp

 // scanner.cpp
 //
-// (C) 2002-2009 MicroNeil Research Corporation
+// (C) 2002-2020 MicroNeil Research Corporation
 
 // 20041117 _M - Included new improved Filter Chain module UrlDecode. This module
 // scans each anchor or image tag for URL encoded characters and converts them to

scanner.hpp

 // scanner.hpp
 //
-// (C) 2002-2009 MicroNeil Research Corporation
+// (C) 2002-2020 MicroNeil Research Corporation
 
 // 20040113 _M - Added Reset() to the scanner object to more completely handle
 // cleanup after processing a message. Where previously the calling code would

snf_saccades.cpp

-// snf_saccades.cpp
-// Saccades engine adaptation for MessageSniffer
-// Copyright 2014 MicroNeil Research Corporation (www.microneil.com)
-// Licensed to ARM Research Labs for use in Message Sniffer.
-
-#include <set>
-#include <vector>
-#include "snf_saccades.hpp"
-
-bool doesOverlap(unsigned int workingStart, unsigned int testStart, unsigned int workingFinish) {
-    return(
-      testStart >= workingStart &&
-      testStart <= workingFinish
-    );
-}
-std::vector<saccade> saccades_engine::recall() {
-    std::vector<saccade> recollection;
-    int markersSize = markers.size();
-    if(0 < markersSize) {
-        std::set<saccade_marker>::iterator i;
-        recollection.reserve(markers.size());
-        bool isFirstPass = true;
-        unsigned int workingStart = 0;
-        unsigned int workingFinish = 0;
-        for(i = markers.begin(); i != markers.end(); i++) {
-            const saccade& theSaccade = (*i).theSaccade;
-            if(isFirstPass) {
-                workingStart = theSaccade.start;
-                workingFinish = theSaccade.finish;
-                isFirstPass = false;
-            }
-            else {
-                if(doesOverlap(workingStart, theSaccade.start, workingFinish)) {
-                    workingFinish = theSaccade.finish;
-                }
-                else {
-                    recollection.push_back(saccade::unstretched(workingStart, workingFinish));
-                    workingStart = theSaccade.start;
-                    workingFinish = theSaccade.finish;
-                }
-            }
-        }
-        recollection.push_back(saccade::unstretched(workingStart, workingFinish));
-    }
-    return recollection;
-}
-
-void saccades_engine::disconnectEngram(int theSlot) {
-    saccade_engram& current = engrams.at(theSlot);
-    saccade_engram& above = engrams.at(current.nextFresh);
-    above.nextStale = current.nextStale;
-
-    if(mostStale == theSlot) {
-        mostStale = current.nextFresh;
-    }
-    else {
-        saccade_engram& below = engrams.at(current.nextStale);
-        below.nextFresh = current.nextFresh;
-    }
-
-    current.nextFresh = saccade_engram::NoneMore;
-    current.nextStale = saccade_engram::NoneMore;
-}
-
-void saccades_engine::connectFreshestEngram(int theSlot) {
-    engrams.at(theSlot).nextStale = mostFresh;
-    if(mostFresh != saccade_engram::NoneMore) {
-        engrams.at(mostFresh).nextFresh = theSlot;
-    }
-    mostFresh = theSlot;
-    if(mostStale == saccade_engram::NoneMore) {
-        mostStale = theSlot;
-    }
-}
-
-void saccades_engine::freshenEngram(int theSlot) {
-    if(mostFresh == theSlot) return;
-    disconnectEngram(theSlot);
-    connectFreshestEngram(theSlot);
-}
-
-void saccades_engine::makeEngram(saccade s) {
-    int newSlot = engrams.size();
-    engrams.push_back(saccade_engram(s));
-    connectFreshestEngram(newSlot);
-    markers.insert(saccade_marker(s, newSlot));
-}
-
-void saccades_engine::recycleEngram(saccade s) {
-    int recycleSlot = mostStale;
-    saccade_marker dropMarker(engrams.at(recycleSlot).theSaccade, recycleSlot);
-    markers.erase(dropMarker);
-    engrams.at(recycleSlot).theSaccade = s;
-    saccade_marker insertMarker(s, recycleSlot);
-    markers.insert(insertMarker);
-    freshenEngram(recycleSlot);
-}
-
-void saccades_engine::evoke(saccade s) {
-    bool stillGrowing = (capacity > engrams.size());
-    std::set<saccade_marker>::iterator i;
-    saccade_marker testMarker(s,0);
-    i = markers.find(testMarker);
-    bool isRemembered = (i != markers.end());
-
-    if(isRemembered) {
-        freshenEngram((*i).theSlot);
-    }
-    else {
-        if(stillGrowing) {
-            makeEngram(s);
-        }
-        else {
-            recycleEngram(s);
-        }
-    }
-}
-
-void saccades_engine::learn(std::vector<saccade>& experiences) {
-    for(unsigned int i = 0; i < experiences.size(); i++) {
-        evoke(experiences[i]);
-    }
-}

snf_saccades.hpp

-// snf_saccades.hpp
-// Saccades engine adaptation for MessageSniffer
-// Copyright 2014-2020 MicroNeil Research Corporation (www.microneil.com)
-// Licensed to ARM Research Labs for use in Message Sniffer.
-
-#pragma once
-
-#include <set>
-#include <vector>
-
-class saccades_engine;
-class saccade {
-  friend class saccades_engine;
-  private:
-    const static unsigned int stretchSize = 8;
-    const static unsigned int stretchMark = stretchSize / 2;
-    const static unsigned int stretchMask = ((~0U) ^ (stretchSize - 1));
-
-    unsigned int stretchLeft(unsigned int s) {
-        s = (stretchMark > s) ? s : (s - (stretchMark));
-        return (s & stretchMask);
-    }
-
-    unsigned int stretchRight(unsigned int f) {
-        f = (start > f) ? start : f;
-        return ((f + stretchSize + stretchMark) & stretchMask);
-    }
-
-    saccade() : start(0), finish(0) {}
-    static saccade unstretched(unsigned int s, unsigned int f) {
-        saccade u;
-        u.start = s;
-        u.finish = f;
-        return u;
-    }
-
-  public:
-    unsigned int start;
-    unsigned int finish;
-
-    saccade(unsigned int s, unsigned int f) :
-      start(stretchLeft(s)),
-      finish(stretchRight(f)) {}
-
-    bool operator<(const saccade& r) const {
-        if(start < r.start) return true;
-        if(start > r.start) return false;
-        return (finish < r.finish);
-    }
-
-    bool operator==(const saccade& r) const {
-        return (
-          start == r.start &&
-          finish == r.finish
-        );
-    }
-};
-
-class saccade_marker {
-  public:
-    const saccade theSaccade;
-    unsigned int theSlot;
-    saccade_marker(saccade key, unsigned int slot) :
-      theSaccade(key),
-      theSlot(slot) {}
-
-    bool operator<(const saccade_marker& r) const {
-        return (theSaccade < r.theSaccade);
-    }
-};
-
-struct saccade_engram {
-    static const int NoneMore = -1;
-    int nextFresh;
-    int nextStale;
-    saccade theSaccade;
-
-    saccade_engram() :
-      nextFresh(NoneMore),
-      nextStale(NoneMore),
-      theSaccade(0,0) {}
-
-    saccade_engram(saccade s) :
-      nextFresh(NoneMore),
-      nextStale(NoneMore),
-      theSaccade(s) {}
-};
-
-class saccades_engine {
-  private:
-    std::vector<saccade_engram> engrams;
-    std::set<saccade_marker> markers;
-
-    const unsigned int capacity;
-    int mostFresh;
-    int mostStale;
-
-    void evoke(saccade s);
-
-    void disconnectEngram(int theSlot);
-    void connectFreshestEngram(int theSlot);
-    void freshenEngram(int theSlot);
-    void makeEngram(saccade s);
-    void recycleEngram(saccade s);
-
-  public:
-
-    saccades_engine(unsigned int c) :
-        capacity(c),
-        mostFresh(saccade_engram::NoneMore),
-        mostStale(saccade_engram::NoneMore) {
-          engrams.reserve(capacity);
-        }
-
-    std::vector<saccade> recall();
-    void learn(std::vector<saccade>& experiences);
-
-};