madscientist
/
SNFMulti
Watch 1
Star 0
Fork 0
Code Issues 0 Pull Requests 0 Releases 0 Wiki Activity
Browse Source

Bumped tiny revision for buffer allocation bug fix in CodeDweller configuration.cpp 

git-svn-id: https://svn.microneil.com/svn/SNFMulti/trunk@66 dc71a809-1921-45c4-985c-09c81d0142d9
wx
madscientist 8 years ago
parent
71c098f26a
commit
304ab5e19b
1 changed files with 205 additions and 205 deletions
Unified View Show Diff Stats
  1. 205
    205
      SNFMulti.cpp

+ 205
- 205
SNFMulti.cpp View File

#include <cstdlib> #include <cstdlib>


#include <sstream> #include <sstream>
#include "SNFMulti.hpp"
#include "snf_saccades.hpp"
#include "SNFMulti.hpp"
#include "snf_saccades.hpp"


#include "../CodeDweller/timing.hpp" #include "../CodeDweller/timing.hpp"




//// Version Info //// Version Info


const char* SNF_ENGINE_VERSION = "SNFMulti Engine Version 3.2.0 Build: " __DATE__ " " __TIME__;
const char* SNF_ENGINE_VERSION = "SNFMulti Engine Version 3.2.1 Build: " __DATE__ " " __TIME__;


//// Script Caller Methods //// Script Caller Methods




TokenMatrix* TryThis = NULL; // We need our candidate to remain in scope. TokenMatrix* TryThis = NULL; // We need our candidate to remain in scope.


try { // This try block decodes the problem.
try { // This try block decodes the problem.
try { // This try block does cleanup work. try { // This try block does cleanup work.
TryThis = new TokenMatrix(); // Grab a new Token Matrix TryThis = new TokenMatrix(); // Grab a new Token Matrix
TryThis->Load(RuleFilePath); // Load it from the provided file path TryThis->Load(RuleFilePath); // Load it from the provided file path
TryThis->Validate(SecurityKey); // Validate it with the provided security key TryThis->Validate(SecurityKey); // Validate it with the provided security key
TryThis->Verify(SecurityKey); // Verify that it is not corrupt.
}
catch(...) { // Clean up after any exceptions.
RefreshInProgress = false; // We're not refreshing now.
if(TryThis) { // If we allocated a TokenMatrix then
delete TryThis; // we need to reclaim the memory
TryThis = 0; // and erase the pointer.
} // With everything nice and clean we can
throw; // rethrow he exception for decoding.
TryThis->Verify(SecurityKey); // Verify that it is not corrupt.
}
catch(...) { // Clean up after any exceptions.
RefreshInProgress = false; // We're not refreshing now.
if(TryThis) { // If we allocated a TokenMatrix then
delete TryThis; // we need to reclaim the memory
TryThis = 0; // and erase the pointer.
} // With everything nice and clean we can
throw; // rethrow he exception for decoding.
} }
} // If nothing threw, we're golden! } // If nothing threw, we're golden!


IPPattern, IPPattern,
AboveBandPattern AboveBandPattern
}; };
// In order to optimize message file reads when header injection is not activated
// we need to look ahead to see if header injection is likely to be turned on when
// we do the scan. This is a short term fix. The better fix might be to perform
// the configuration load prior to scanning the message -- but that is a much larger
// refactoring that ties up configuration and rulebase resources for a longer time.
// Instead we're going to take an optimistic route and just peek at the configuration.
// If the configuration changes while we're loading the file to be scanned then
// we have two cases. If we go from XHDRInject off to XHDRInject on then we will
// miss adding headers to the message - not a bad outcome. If we go from XHDRInject
// on to XHDRInject off then we might emit headers for an extra message - also not
// a bad outcome.
bool snf_RulebaseHandler::testXHDRInjectOn() {
// In order to optimize message file reads when header injection is not activated
// we need to look ahead to see if header injection is likely to be turned on when
// we do the scan. This is a short term fix. The better fix might be to perform
// the configuration load prior to scanning the message -- but that is a much larger
// refactoring that ties up configuration and rulebase resources for a longer time.
// Instead we're going to take an optimistic route and just peek at the configuration.
// If the configuration changes while we're loading the file to be scanned then
// we have two cases. If we go from XHDRInject off to XHDRInject on then we will
// miss adding headers to the message - not a bad outcome. If we go from XHDRInject
// on to XHDRInject off then we might emit headers for an extra message - also not
// a bad outcome.
bool snf_RulebaseHandler::testXHDRInjectOn() {
ScopeMutex HoldStillPlease(MyMutex); // Lock the rulebase until we're done. ScopeMutex HoldStillPlease(MyMutex); // Lock the rulebase until we're done.
snfCFGData* myCFG = MyCFGmgr.ActiveConfiguration(); // Grab the active configuration. snfCFGData* myCFG = MyCFGmgr.ActiveConfiguration(); // Grab the active configuration.
bool myXHDRInjectOnFlag = (LogOutputMode_Inject == myCFG->XHDROutput_Mode); // True if output mode is inject.
return myXHDRInjectOnFlag; // return the result.
}
bool myXHDRInjectOnFlag = (LogOutputMode_Inject == myCFG->XHDROutput_Mode); // True if output mode is inject.
return myXHDRInjectOnFlag; // return the result.
}


int snf_EngineHandler::scanMessageFile( // Scan this message file. int snf_EngineHandler::scanMessageFile( // Scan this message file.
const string MessageFilePath, // -- this is the file path (and id) const string MessageFilePath, // -- this is the file path (and id)
); );
throw FileError("snf_EngineHandler::scanMessageFile() FileEmpty!"); throw FileError("snf_EngineHandler::scanMessageFile() FileEmpty!");
} }
bool isXHeaderInjectionOn = MyRulebase->testXHDRInjectOn();
bool noNeedToReadFullFile = (false == isXHeaderInjectionOn);
if(noNeedToReadFullFile) {
MessageFileSize = min(MessageFileSize, snf_ScanHorizon);
}
bool isXHeaderInjectionOn = MyRulebase->testXHDRInjectOn();
bool noNeedToReadFullFile = (false == isXHeaderInjectionOn);
if(noNeedToReadFullFile) {
MessageFileSize = min(MessageFileSize, snf_ScanHorizon);
}


vector<unsigned char> MessageBuffer; // Allocate a buffer and size vector<unsigned char> MessageBuffer; // Allocate a buffer and size
try { MessageBuffer.resize(MessageFileSize, 0); } // it to fit the message. try { MessageBuffer.resize(MessageFileSize, 0); } // it to fit the message.
// The insertion point will be at the end of the existing headers. // The insertion point will be at the end of the existing headers.
// We pick that point to be right between the two <cr><lf> so that // We pick that point to be right between the two <cr><lf> so that
// the first blank line will appear at the end of our headers. // the first blank line will appear at the end of our headers.
// We accommodate either <cr><lf> or <lf> line endings.
// We are careful not to search past the end of unreasonably short
// message files.
// We accommodate either <cr><lf> or <lf> line endings.
// We are careful not to search past the end of unreasonably short
// message files.


unsigned int InsertPoint = 0; // Find the insertion point. unsigned int InsertPoint = 0; // Find the insertion point.
bool UseLFOnly = false; // Use \n line endings in files? bool UseLFOnly = false; // Use \n line endings in files?
bool CRLFPresent = false; // Detected \r\n pairs?
unsigned int BiggestPatternSize = 4; // How far we look ahead.
bool BigEnoughMessage = BiggestPatternSize < MessageBuffer.size();
if(BigEnoughMessage){
unsigned int Limit = MessageBuffer.size() - BiggestPatternSize;
bool DataWasSkipped = MessageBuffer.size() > MyScanData.ScanSize;
unsigned int i = 0;
if(DataWasSkipped) { // If our scanner skipped data at
i = MessageBuffer.size() - MyScanData.ScanSize; // the top of the message buffer then
} // we will skip it too.
bool CRLFPresent = false; // Detected \r\n pairs?
unsigned int BiggestPatternSize = 4; // How far we look ahead.
bool BigEnoughMessage = BiggestPatternSize < MessageBuffer.size();
if(BigEnoughMessage){
unsigned int Limit = MessageBuffer.size() - BiggestPatternSize;
bool DataWasSkipped = MessageBuffer.size() > MyScanData.ScanSize;
unsigned int i = 0;
if(DataWasSkipped) { // If our scanner skipped data at
i = MessageBuffer.size() - MyScanData.ScanSize; // the top of the message buffer then
} // we will skip it too.
for(; i < Limit; i++) { // Search for the first blank line. for(; i < Limit; i++) { // Search for the first blank line.


if( // Detect CRLF pairs if present. if( // Detect CRLF pairs if present.
break; // injected header line ends. break; // injected header line ends.
} }
} }
}
}


// Here we must interpret the results of our search. Do we know where // Here we must interpret the results of our search. Do we know where
// our insert point is or do we punt and use the top of the message? // our insert point is or do we punt and use the top of the message?


const LogicFault FaultBadMessageBuffer1("snf_EngineHandler::scanMessage():FaultBadMessageBuffer1(NULL == inputMessageBuffer)"); const LogicFault FaultBadMessageBuffer1("snf_EngineHandler::scanMessage():FaultBadMessageBuffer1(NULL == inputMessageBuffer)");
const LogicFault FaultBadMessageBuffer2("snf_EngineHandler::scanMessage():FaultBadMessageBuffer2(0 >= inputMessageLength)"); const LogicFault FaultBadMessageBuffer2("snf_EngineHandler::scanMessage():FaultBadMessageBuffer2(0 >= inputMessageLength)");
const char Unknown_SNFMatchFlag = '-';
const char Panic_SNFMatchFlag = 'p';
const char Match_SNFMatchFlag = 'm';
const char White_SNFMatchFlag = 'w';
const char Final_SNFMatchFlag = 'f';
void captureMatchRecord(snf_match& M, MatchRecord* R) {
M.flag = Unknown_SNFMatchFlag;
M.ruleid = R->RuleId();
M.symbol = R->RuleGroup();
M.index = R->MatchStartPosition;
M.endex = R->MatchEndPosition;
}
void snf_SaccadesHandler::applySaccades(EvaluationMatrix* Scanner, vector<unsigned char>& Data) {
if(NULL == Scanner) return;
bool isTimeToPeek = (0 >= TimeToPeekCounter);
if(isTimeToPeek) {
TimeToPeekCounter = TimeToPeekReset;
return;
} else {
--TimeToPeekCounter;
}
vector<saccade> Saccades = grabSaccades();
for(vector<saccade>::iterator i = Saccades.begin(); i != Saccades.end(); i++) {
const saccade& s = (*i);
if(s.start >= Data.size()) break;
Scanner->evaluateSegment(Data, s.start, s.finish);
}
}
bool isLearnableMatch(MatchRecord* m) {
bool isGoodSymbol = (0 <= m->RuleGroup() && 64 > m->RuleGroup());
bool isBeyondAlwaysScan = (snf_SaccadesHandler::AlwaysScanLength < m->MatchEndPosition);
return (isGoodSymbol && isBeyondAlwaysScan);
}
void snf_SaccadesHandler::learnMatches(MatchRecord* Matches) {
if(NULL == Matches) return;
vector<saccade> MatchesToLearn;
saccade WatchForHeaderWhiteRules(0, AlwaysScanLength);
MatchesToLearn.push_back(WatchForHeaderWhiteRules);
for(MatchRecord* m = Matches; NULL != m; m = m->NextMatchRecord) {
if(isLearnableMatch(m)) {
MatchesToLearn.push_back(
saccade(
m->MatchStartPosition,
m->MatchEndPosition)
);
}
}
if(0 < MatchesToLearn.size()) {
lockAndLearn(MatchesToLearn);
}
}
static snf_SaccadesHandler SaccadeBrain;
static snf_IPStrangerList StrangersList;
const char Unknown_SNFMatchFlag = '-';
const char Panic_SNFMatchFlag = 'p';
const char Match_SNFMatchFlag = 'm';
const char White_SNFMatchFlag = 'w';
const char Final_SNFMatchFlag = 'f';
void captureMatchRecord(snf_match& M, MatchRecord* R) {
M.flag = Unknown_SNFMatchFlag;
M.ruleid = R->RuleId();
M.symbol = R->RuleGroup();
M.index = R->MatchStartPosition;
M.endex = R->MatchEndPosition;
}
void snf_SaccadesHandler::applySaccades(EvaluationMatrix* Scanner, vector<unsigned char>& Data) {
if(NULL == Scanner) return;
bool isTimeToPeek = (0 >= TimeToPeekCounter);
if(isTimeToPeek) {
TimeToPeekCounter = TimeToPeekReset;
return;
} else {
--TimeToPeekCounter;
}
vector<saccade> Saccades = grabSaccades();
for(vector<saccade>::iterator i = Saccades.begin(); i != Saccades.end(); i++) {
const saccade& s = (*i);
if(s.start >= Data.size()) break;
Scanner->evaluateSegment(Data, s.start, s.finish);
}
}
bool isLearnableMatch(MatchRecord* m) {
bool isGoodSymbol = (0 <= m->RuleGroup() && 64 > m->RuleGroup());
bool isBeyondAlwaysScan = (snf_SaccadesHandler::AlwaysScanLength < m->MatchEndPosition);
return (isGoodSymbol && isBeyondAlwaysScan);
}
void snf_SaccadesHandler::learnMatches(MatchRecord* Matches) {
if(NULL == Matches) return;
vector<saccade> MatchesToLearn;
saccade WatchForHeaderWhiteRules(0, AlwaysScanLength);
MatchesToLearn.push_back(WatchForHeaderWhiteRules);
for(MatchRecord* m = Matches; NULL != m; m = m->NextMatchRecord) {
if(isLearnableMatch(m)) {
MatchesToLearn.push_back(
saccade(
m->MatchStartPosition,
m->MatchEndPosition)
);
}
}
if(0 < MatchesToLearn.size()) {
lockAndLearn(MatchesToLearn);
}
}
static snf_SaccadesHandler SaccadeBrain;
static snf_IPStrangerList StrangersList;
int snf_EngineHandler::scanMessage( // Scan this message (in buffer). int snf_EngineHandler::scanMessage( // Scan this message (in buffer).
const unsigned char* inputMessageBuffer, // -- this is the message buffer. const unsigned char* inputMessageBuffer, // -- this is the message buffer.
const int inputMessageLength, // -- this is the length of the buffer. const int inputMessageLength, // -- this is the length of the buffer.
// data, as well as the ability to output the pre-filtered data for use in // data, as well as the ability to output the pre-filtered data for use in
// rule development and debugging. // rule development and debugging.


DebugInfo = "scanMessage() IZ.GetByte() ==> FilteredData"; // If we panic we are here.
DebugInfo = "scanMessage() IZ.GetByte() ==> FilteredData"; // If we panic we are here.
unsigned char xb=0; unsigned char xb=0;
MyScanData.FilteredData.clear(); // Clear the FilteredData buffer. MyScanData.FilteredData.clear(); // Clear the FilteredData buffer.
try { // Watch for exceptions and scan try { // Watch for exceptions and scan
// If something goes wrong, an exception will be thrown. // If something goes wrong, an exception will be thrown.


DebugInfo = "scanMessage() EvaluateThis(FilteredData)"; // If we panic, here we are. DebugInfo = "scanMessage() EvaluateThis(FilteredData)"; // If we panic, here we are.
if(false == MyScanData.GBUdbTruncateExecuted) { // If we haven't already truncated: if(false == MyScanData.GBUdbTruncateExecuted) { // If we haven't already truncated:
//for(int a = 0, b = MyScanData.FilteredData.size(); a < b; a++) // Scan through the filtered data one //for(int a = 0, b = MyScanData.FilteredData.size(); a < b; a++) // Scan through the filtered data one
// CurrentMatrix->EvaluateThis(MyScanData.FilteredData[a]); // byte at a time.
unsigned int fullLength = MyScanData.FilteredData.size();
SaccadeBrain.applySaccades(CurrentMatrix, MyScanData.FilteredData);
bool messageNotRecognized = (NULL == CurrentMatrix->ResultList);
if(messageNotRecognized) {
CurrentMatrix->evaluateSegment(MyScanData.FilteredData, 0, fullLength);
SaccadeBrain.learnMatches(CurrentMatrix->ResultList);
// CurrentMatrix->EvaluateThis(MyScanData.FilteredData[a]); // byte at a time.
unsigned int fullLength = MyScanData.FilteredData.size();
SaccadeBrain.applySaccades(CurrentMatrix, MyScanData.FilteredData);
bool messageNotRecognized = (NULL == CurrentMatrix->ResultList);
if(messageNotRecognized) {
CurrentMatrix->evaluateSegment(MyScanData.FilteredData, 0, fullLength);
SaccadeBrain.learnMatches(CurrentMatrix->ResultList);
} }
} }


int S = NO_SYMBOL; // so we start there and work down. int S = NO_SYMBOL; // so we start there and work down.


snf_match TmpSNFMatch; // We'll need a buffer for our matches. snf_match TmpSNFMatch; // We'll need a buffer for our matches.
while(NULL!=ResultCursor) { // While we have records to process...
captureMatchRecord(TmpSNFMatch, ResultCursor); // grab the next record and evaluate it.
// Mitigate short-match rulebase events to prevent false positives.
const size_t minimumPatternLength = 5; // Establish a minimum match length.
size_t matchSpan = (TmpSNFMatch.endex - TmpSNFMatch.index); // Determine the length of this match.
bool isShortMatchEvent = (minimumPatternLength > matchSpan); // Identify short-match events.
bool isPanickedRule = ( // In addition to rule IDs that are
MyCFGPacket.isRulePanic(TmpSNFMatch.ruleid) || // in the rule-panic list also treat
isShortMatchEvent // short match events as panic rules.
);
bool isVotingCandidate = (false == isPanickedRule); // Panic rules can't vote.
bool isWhiteRule = (
MyCFGPacket.Config()->TrainingWhiteRuleHandler.isListed(TmpSNFMatch.ruleid) ||
0 == TmpSNFMatch.symbol
);
bool isBestResultCode = (TmpSNFMatch.symbol < S);
// Set an appropriate flag.
if(isPanickedRule) TmpSNFMatch.flag = Panic_SNFMatchFlag;
else if(isWhiteRule) TmpSNFMatch.flag = White_SNFMatchFlag;
else TmpSNFMatch.flag = Match_SNFMatchFlag;
// Vote for best rule match.
if(isVotingCandidate && isBestResultCode) {
FinalResult = ResultCursor;
S = TmpSNFMatch.symbol;
}
// Record this MatchRecord and mMove on to next result.
MyScanData.MatchRecords.push_back(TmpSNFMatch);
ResultsCount++;
ResultCursor=ResultCursor->NextMatchRecord;
while(NULL!=ResultCursor) { // While we have records to process...
captureMatchRecord(TmpSNFMatch, ResultCursor); // grab the next record and evaluate it.
// Mitigate short-match rulebase events to prevent false positives.
const size_t minimumPatternLength = 5; // Establish a minimum match length.
size_t matchSpan = (TmpSNFMatch.endex - TmpSNFMatch.index); // Determine the length of this match.
bool isShortMatchEvent = (minimumPatternLength > matchSpan); // Identify short-match events.
bool isPanickedRule = ( // In addition to rule IDs that are
MyCFGPacket.isRulePanic(TmpSNFMatch.ruleid) || // in the rule-panic list also treat
isShortMatchEvent // short match events as panic rules.
);
bool isVotingCandidate = (false == isPanickedRule); // Panic rules can't vote.
bool isWhiteRule = (
MyCFGPacket.Config()->TrainingWhiteRuleHandler.isListed(TmpSNFMatch.ruleid) ||
0 == TmpSNFMatch.symbol
);
bool isBestResultCode = (TmpSNFMatch.symbol < S);
// Set an appropriate flag.
if(isPanickedRule) TmpSNFMatch.flag = Panic_SNFMatchFlag;
else if(isWhiteRule) TmpSNFMatch.flag = White_SNFMatchFlag;
else TmpSNFMatch.flag = Match_SNFMatchFlag;
// Vote for best rule match.
if(isVotingCandidate && isBestResultCode) {
FinalResult = ResultCursor;
S = TmpSNFMatch.symbol;
}
// Record this MatchRecord and mMove on to next result.
MyScanData.MatchRecords.push_back(TmpSNFMatch);
ResultsCount++;
ResultCursor=ResultCursor->NextMatchRecord;
} }
if(NO_SYMBOL != S) { // If a pattern match was detected then if(NO_SYMBOL != S) { // If a pattern match was detected then
MyScanData.PatternWasFound = true; // trip the flag and record the MyScanData.PatternWasFound = true; // trip the flag and record the
MyScanData.PatternID = FinalResult->RuleId(); // Rule ID and the MyScanData.PatternID = FinalResult->RuleId(); // Rule ID and the
) { ) {


// GBUdb training is enabled. // GBUdb training is enabled.
bool discoveredNewIP = false;
IP4Address theSourceIP = MyScanData.SourceIPRecord().IP;
bool discoveredNewIP = false;
IP4Address theSourceIP = MyScanData.SourceIPRecord().IP;
switch(ScanResultType) { // Evaluate the scan result. switch(ScanResultType) { // Evaluate the scan result.
case NoPattern: // On no pattern (benefit of doubt) or case NoPattern: // On no pattern (benefit of doubt) or
case WhitePattern: { // a white pattern:
case WhitePattern: { // a white pattern:
GBUdbRecord thisRecord = // Grab the GBUdb record for later GBUdbRecord thisRecord = // Grab the GBUdb record for later
MyRulebase->MyGBUdb.addGood( // then add a good count to the MyRulebase->MyGBUdb.addGood( // then add a good count to the
theSourceIP); // source IP.
discoveredNewIP = (0 == thisRecord.Bad() && 1 == thisRecord.Good());
if(discoveredNewIP) { // New IPs are strangers.
StrangersList.addStranger(theSourceIP); // Add them to the list
thisRecord.Bad(thisRecord.Good()); // and set their reputation
MyRulebase->MyGBUdb.setRecord(theSourceIP, thisRecord); // to 50/50 at best.
} else
if( // Known IPs that are getting
thisRecord.Good() > thisRecord.Bad() && // an advantage but are on the
StrangersList.isStranger(theSourceIP) // strangers list get put back
) { // to a 50/50 reputation.
unsigned int equalizationValue = thisRecord.Good();
if(1 < equalizationValue) equalizationValue = equalizationValue / 2;
thisRecord.Bad(equalizationValue);
thisRecord.Good(equalizationValue);
MyRulebase->MyGBUdb.setRecord(theSourceIP, thisRecord);
}
theSourceIP); // source IP.
discoveredNewIP = (0 == thisRecord.Bad() && 1 == thisRecord.Good());
if(discoveredNewIP) { // New IPs are strangers.
StrangersList.addStranger(theSourceIP); // Add them to the list
thisRecord.Bad(thisRecord.Good()); // and set their reputation
MyRulebase->MyGBUdb.setRecord(theSourceIP, thisRecord); // to 50/50 at best.
} else
if( // Known IPs that are getting
thisRecord.Good() > thisRecord.Bad() && // an advantage but are on the
StrangersList.isStranger(theSourceIP) // strangers list get put back
) { // to a 50/50 reputation.
unsigned int equalizationValue = thisRecord.Good();
if(1 < equalizationValue) equalizationValue = equalizationValue / 2;
thisRecord.Bad(equalizationValue);
thisRecord.Good(equalizationValue);
MyRulebase->MyGBUdb.setRecord(theSourceIP, thisRecord);
}
break; break;
}
}

case BlackPattern: { // On a black pattern:


case BlackPattern: { // On a black pattern:
GBUdbRecord thisRecord = // Grab the GBUdb record for later GBUdbRecord thisRecord = // Grab the GBUdb record for later
MyRulebase->MyGBUdb.addBad( // Add a bad count to the source IP MyRulebase->MyGBUdb.addBad( // Add a bad count to the source IP
MyScanData.SourceIPRecord().IP); // in the GBUdb.
discoveredNewIP = (1 == thisRecord.Bad() && 0 == thisRecord.Good());
if(discoveredNewIP) StrangersList.addStranger(theSourceIP);
MyScanData.SourceIPRecord().IP); // in the GBUdb.
discoveredNewIP = (1 == thisRecord.Bad() && 0 == thisRecord.Good());
if(discoveredNewIP) StrangersList.addStranger(theSourceIP);


break; break;
} }

Write Preview
Loading…
Cancel
Save