Nevar pievienot vairāk kā 25 tēmas Tēmai ir jāsākas ar burtu vai ciparu, tā var saturēt domu zīmes ('-') un var būt līdz 35 simboliem gara.

SNFMulti.cpp 142KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315
  1. // SNFMulti.cpp
  2. //
  3. // (C) Copyright 2006 - 2009 ARM Research Labs, LLC
  4. // See www.armresearch.com for the copyright terms.
  5. //
  6. // 20060121_M
  7. //
  8. // See SNFMulti.hpp for history and detailed notes.
  9. #include <sys/types.h>
  10. #include <sys/stat.h>
  11. #include <ctime>
  12. #include <cstring>
  13. #include <cstdlib>
  14. #include <sstream>
  15. #include "SNFMulti.hpp"
  16. #include "snf_saccades.hpp"
  17. #include "../CodeDweller/timing.hpp"
  18. //#include "../nvwa-0.6/nvwa/debug_new.h"
  19. namespace cd = codedweller;
  20. //// Version Info
  21. const char* SNF_ENGINE_VERSION = "SNFMulti Engine Version 3.2.1 Build: " __DATE__ " " __TIME__;
  22. //// Script Caller Methods
  23. const ThreadType ScriptCaller::Type("Script Caller"); // Script caller thread type mnemonic.
  24. const ThreadState ScriptCaller::CallingSystem("In system()"); // Script caller "CallingSystem" state.
  25. const ThreadState ScriptCaller::PendingGuardTime("Guard Time"); // Script caller "GuardTime" state.
  26. const ThreadState ScriptCaller::StandingBy("Standby"); // Script caller "Standby" state.
  27. const ThreadState ScriptCaller::Disabled("Disabled"); // State when unable to run.
  28. const int ScriptGuardDefault = 180000; // 3 Minute Default Guard Time.
  29. ScriptCaller::ScriptCaller(string S) : // Script caller constructor (with name).
  30. Thread(ScriptCaller::Type, S), // Set up the thread type and name.
  31. GuardTimer(ScriptGuardDefault), // Initialize the guard time.
  32. GoFlag(false), // Not ready to go yet.
  33. DieFlag(false), // Not ready to die yet.
  34. myLastResult(0) { // No last result yet.
  35. run(); // Launch the thread.
  36. }
  37. ScriptCaller::~ScriptCaller() { // Destructor.
  38. DieFlag = true; // Set the die flag.
  39. Sleeper WaitATic(1000); // One second sleeper.
  40. for(int x = 10; x > 0; x--) { // We don't join, we might get stuck.
  41. if(false == isRunning()) break; // If we're still running then wait
  42. WaitATic(); // up to 10 seconds, then just exit.
  43. } // If the thread is stuck it will
  44. } // just get closed.
  45. string ScriptCaller::ScriptToRun() { // Safely grab the SystemCallText.
  46. ScopeMutex Freeze(MyMutex); // Protect the string.
  47. return SystemCallText; // Grab a copy of the text.
  48. }
  49. bool ScriptCaller::hasGuardExpired() { // True if guard time has expired.
  50. ScopeMutex Freeze(MyMutex); // Protect the timer.
  51. return GuardTimer.isExpired(); // If it has expired we're true.
  52. }
  53. void ScriptCaller::SystemCall(string S) { // Set the SystemCall text.
  54. ScopeMutex Freeze(MyMutex); // Protect the string object.
  55. SystemCallText = S; // Set it's data.
  56. }
  57. const int MinimumGuardTime = 60000; // Minimum Guard Time 1 minute.
  58. void ScriptCaller::GuardTime(int T) { // Set the Guard Time.
  59. if(MinimumGuardTime > T) T = MinimumGuardTime; // Enforce our lower limit.
  60. ScopeMutex Freeze(MyMutex); // Protect the Guard Timer.
  61. GuardTimer.setDuration(T); // Set the duration.
  62. GuardTimer.restart(); // Restart the timer.
  63. }
  64. void ScriptCaller::trigger() { // Trigger the system() call.
  65. GoFlag = true; // Set the flag.
  66. }
  67. int ScriptCaller::LastResult() { // Return the result code from
  68. return myLastResult; // the last system() call.
  69. }
  70. void ScriptCaller::myTask() { // Safely call system() when triggered.
  71. Sleeper WaitATic(1000); // One second sleeper.
  72. while(false == DieFlag) { // While it's not time to die:
  73. WaitATic(); // Pause for 1 sec each round.
  74. string ScriptThisRound = ScriptToRun(); // Grab the current script.
  75. if(0 < ScriptToRun().length()) { // If script text is defined and
  76. if(true == GoFlag) { // If GoFlag is triggered and
  77. if(hasGuardExpired()) { // Guard time is expired:
  78. CurrentThreadState(CallingSystem); // Publish our state.
  79. myLastResult = system(ScriptThisRound.c_str()); // Make the system call.
  80. GoFlag = false; // Done with that trigger.
  81. GuardTimer.restart(); // Restart our Guard Time.
  82. } else { // If we're waiting for Guard Time:
  83. CurrentThreadState(PendingGuardTime); // publish that state and hold down
  84. GoFlag = false; // the trigger signal (no stale go).
  85. }
  86. } else { // If nothing is triggered yet then
  87. CurrentThreadState(StandingBy); // we are standing by.
  88. }
  89. } else { // If we have no script to run then
  90. CurrentThreadState(Disabled); // we are disabled.
  91. }
  92. }
  93. }
  94. //// Rulebase Reloader Methods
  95. // How to get timestamps on critical files.
  96. time_t getFileTimestamp(string FileName) {
  97. struct stat FileNameStat; // First we need a stat buffer.
  98. if(0 != stat(FileName.c_str(), &FileNameStat)) { // If we can't get the stat we
  99. return 0; // will return 0;
  100. } // If all goes well we return
  101. return FileNameStat.st_mtime; // the last modified time_t.
  102. }
  103. void snf_Reloader::captureFileStats() { // Get stats for later comparison.
  104. snfCFGData& C = *(MyRulebase.MyCFGmgr.ActiveConfiguration()); // Reference the active config.
  105. RulebaseFileCheckName = C.RuleFilePath; // Build/Get Rulebase File Name.
  106. ConfigFileCheckName = C.ConfigFilePath; // Build/Get Configuration File Name.
  107. IgnoreListCheckFileName = C.paths_workspace_path; // Build/Get Ignore File Name.
  108. IgnoreListCheckFileName.append("GBUdbIgnoreList.txt");
  109. RulebaseFileTimestamp = getFileTimestamp(RulebaseFileCheckName); // Timestamps to check for
  110. ConfigurationTimestamp = getFileTimestamp(ConfigFileCheckName); // changes in configuration data
  111. IgnoreListTimestamp = getFileTimestamp(IgnoreListCheckFileName); // or rulebase files.
  112. }
  113. bool snf_Reloader::StatsAreDifferent() { // Check file stats for changes.
  114. return ( // Return true if any of the
  115. RulebaseFileTimestamp != getFileTimestamp(RulebaseFileCheckName) || // Rulebase File, or the
  116. ConfigurationTimestamp != getFileTimestamp(ConfigFileCheckName) || // Configuration File, or the
  117. IgnoreListTimestamp != getFileTimestamp(IgnoreListCheckFileName) // Ignore List File have changed.
  118. );
  119. }
  120. const int MSPerSec = 1000; // 1000 milliseconds per second.
  121. void snf_Reloader::captureGetterConfig() { // Update RulebaseGetter config.
  122. snfCFGData& C = *(MyRulebase.MyCFGmgr.ActiveConfiguration()); // Reference the active config.
  123. RulebaseGetterIsTurnedOn = ( // Is the script caller on or off?
  124. true == C.update_script_on_off && // We're on if the bit is set and
  125. 0 < C.update_script_call.length() // we have a non-empty script to call.
  126. );
  127. if(RulebaseGetterIsTurnedOn) { // If it is turned on:
  128. RulebaseGetter.SystemCall(C.update_script_call); // Set the script call and
  129. RulebaseGetter.GuardTime(C.update_script_guard_time * MSPerSec); // the cycle guard time.
  130. }
  131. else { // If the scripter is turned off:
  132. RulebaseGetter.SystemCall(""); // Set the script to nothing.
  133. }
  134. }
  135. const string snfReloadContext = "--RELOADING--"; // Context for info and error logs.
  136. void snf_Reloader::myTask() { // How do we do this refresh thing?
  137. Sleeper WaitATic(1000); // Wait a second between checks.
  138. while(!TimeToStop) { // While it's not time to stop:
  139. if(
  140. RulebaseGetterIsTurnedOn && // If our rulebase getter is enabled
  141. MyRulebase.MyLOGmgr.isUpdateAvailable() // and a new rulebase is availalbe:
  142. ) {
  143. RulebaseGetter.trigger(); // Trigger the update script (if any).
  144. }
  145. if(StatsAreDifferent()) { // Check the stats. If different:
  146. try { // safely attempt a reload.
  147. WaitATic(); // Wait a tic to let things stabilize
  148. MyRulebase.refresh(); // then call refresh on the handler.
  149. captureFileStats(); // If it works, capture the new stats.
  150. captureGetterConfig(); // Also update the RulebaseGetter.
  151. MyRulebase.logThisInfo( // Log our success.
  152. snfReloadContext, snf_SUCCESS, "Success");
  153. }
  154. catch(const snf_RulebaseHandler::IgnoreListError&) { // If we get an IgnoreListError - say so.
  155. MyRulebase.logThisError(
  156. snfReloadContext, snf_ERROR_RULE_FILE, "IgnoreListError");
  157. }
  158. catch(const snf_RulebaseHandler::ConfigurationError&) { // If we get a ConfigurationError - say so.
  159. MyRulebase.logThisError(
  160. snfReloadContext, snf_ERROR_RULE_FILE, "ConfigurationError");
  161. }
  162. catch(const snf_RulebaseHandler::FileError&) { // If we get a FileError - say so.
  163. MyRulebase.logThisError(
  164. snfReloadContext, snf_ERROR_RULE_FILE, "FileError");
  165. }
  166. catch(const snf_RulebaseHandler::AuthenticationError&) { // If we get a Auth Error - say so.
  167. MyRulebase.logThisError(
  168. snfReloadContext, snf_ERROR_RULE_AUTH, "AuthError");
  169. }
  170. catch(const snf_RulebaseHandler::Busy&) { // If we get a Busy Exception - say so.
  171. MyRulebase.logThisError(
  172. snfReloadContext, snf_ERROR_UNKNOWN, "BusyError");
  173. }
  174. catch(const snf_RulebaseHandler::Panic&) { // If we get a Panic - say so.
  175. MyRulebase.logThisError(
  176. snfReloadContext, snf_ERROR_UNKNOWN, "PanicError");
  177. }
  178. catch(...) { // If we get some other error - shout!
  179. MyRulebase.logThisError(
  180. snfReloadContext, snf_ERROR_UNKNOWN, "UnhandledError");
  181. }
  182. }
  183. WaitATic(); // Wait before the next loop.
  184. }
  185. }
  186. const ThreadType snf_Reloader::Type("snf_Reloader"); // The thread's type.
  187. snf_Reloader::snf_Reloader(snf_RulebaseHandler& R) : // When we are created, we
  188. Thread(snf_Reloader::Type, "Reloader"), // brand and name our thread.
  189. MyRulebase(R), // Capture the rulebase handler.
  190. TimeToStop(false), // It's not time to stop yet.
  191. RulebaseGetter("RulebaseGetter"), // Setup our ScriptCaller thread.
  192. RulebaseGetterIsTurnedOn(false) { // Rulebase getter is off at first.
  193. captureFileStats(); // Set up the initial stats.
  194. captureGetterConfig(); // Set up RulebaseGetter config.
  195. run(); // Run our maintenenace thread.
  196. }
  197. snf_Reloader::~snf_Reloader() { // When we are destroyed we
  198. TimeToStop = true; // set our time to stop bit
  199. join(); // and wait for the thread.
  200. }
  201. //// snfCFGPacket Methods
  202. snfCFGPacket::snfCFGPacket(snf_RulebaseHandler* R) : // When we are created:
  203. MyRulebase(R), // Capture our rulebase handler and
  204. MyTokenMatrix(NULL), // ready our token matrix and
  205. MyCFGData(NULL) { // cfg pointers.
  206. if(MyRulebase) { MyRulebase->grab(*this); } // Safely grab our rulebase.
  207. }
  208. snfCFGPacket::~snfCFGPacket() { if(MyRulebase) MyRulebase->drop(*this); } // Safely drop our rulebase when we die.
  209. TokenMatrix* snfCFGPacket::Tokens() { return MyTokenMatrix; } // Consumers read the Token Matrix and
  210. snfCFGData* snfCFGPacket::Config() { return MyCFGData; } // the snfCFGData.
  211. bool snfCFGPacket::bad() { // If anything is missing it's not good.
  212. return (NULL == MyTokenMatrix || NULL == MyCFGData); // True if any of these aren NULL.
  213. }
  214. bool snfCFGPacket::isRulePanic(int R) { // Test for a rule panic.
  215. return(RulePanics.end() != RulePanics.find(R)); // Find it in the list, it's a panic.
  216. }
  217. //// Rulebase Handler Methods
  218. snf_RulebaseHandler::~snf_RulebaseHandler(){ // Destruct the handler.
  219. close(); // Close before we go.
  220. }
  221. bool snf_RulebaseHandler::isReady(){ // Is the object ready?
  222. return (NULL!=Rulebase); // Have Rulebase? We're ready.
  223. }
  224. bool snf_RulebaseHandler::isBusy(){ // Is a refresh/open in progress or
  225. return (RefreshInProgress || 0<RetiringCount); // an older rulebase is not yet retired.
  226. }
  227. int snf_RulebaseHandler::getReferenceCount(){ // How many Engines using this handler.
  228. return ReferenceCount; // Tell them the count bob.
  229. }
  230. int snf_RulebaseHandler::getCurrentCount(){ // How many Engines active in the current rb.
  231. return CurrentCount; // Tell them what it is bob.
  232. }
  233. int snf_RulebaseHandler::getRetiringCount(){ // How many Engines active in the old rb.
  234. return RetiringCount; // Tell them what it is bob.
  235. }
  236. // FileUTC(FileName) - utility function for tagging the active rulebase
  237. cd::RuntimeCheck FileUTCGoodTimestampLength("SNFMulti.cpp:FileUTC snprintf(...) == CorrectTimestampLength");
  238. string FileUTC(string FileName) { // Gets a files UTC.
  239. struct stat FileNameStat; // First we need a stat buffer.
  240. string t; // We also need a Timestamp holder.
  241. if(0 != stat(FileName.c_str(), &FileNameStat)) { // If we can't get the stat we
  242. t.append("00000000000000"); return t; // will return all zeroz to
  243. } // make sure we should get the file.
  244. struct tm FileNameTime; // Allocate a time structure.
  245. FileNameTime = *(gmtime(&FileNameStat.st_mtime)); // Copy the file time to it as UTC.
  246. char TimestampBfr[16]; // Timestamp buffer.
  247. size_t l = snprintf( // Format yyyymmddhhmmss
  248. TimestampBfr, sizeof(TimestampBfr),
  249. "%04d%02d%02d%02d%02d%02d",
  250. FileNameTime.tm_year+1900,
  251. FileNameTime.tm_mon+1,
  252. FileNameTime.tm_mday,
  253. FileNameTime.tm_hour,
  254. FileNameTime.tm_min,
  255. FileNameTime.tm_sec
  256. );
  257. const size_t CorrectTimestampLength = 4+2+2+2+2+2;
  258. FileUTCGoodTimestampLength(l == CorrectTimestampLength);
  259. t.append(TimestampBfr); // Append the timestamp to t
  260. return t; // and return it to the caller.
  261. }
  262. // Auto Reload Controls
  263. bool snf_RulebaseHandler::AutoRefresh(bool On) { // Turn on/off auto refresh.
  264. if(On) { // If they want Reload On:
  265. if(!AutoRefresh()) { // and it isn't already on:
  266. try { MyReloader = new snf_Reloader(*this); } // try to set up a Reloader.
  267. catch(...) { MyReloader = 0; } // If that fails we don't
  268. } // have one. If it's already
  269. } // on do nothing.
  270. else { // If they want Reload Off:
  271. if(AutoRefresh()) { // and it is turned on:
  272. delete MyReloader; // destroy the reloader and
  273. MyReloader = 0; // zero it's pointer.
  274. }
  275. }
  276. return AutoRefresh(); // Return the truth (on/off)
  277. }
  278. bool snf_RulebaseHandler::AutoRefresh() { // True if AutoRefresh is on.
  279. return (0 != MyReloader); // If we have one, it's on.
  280. }
  281. // _snf_LoadNewRulebase()
  282. // This is actually a common sub-funtion. It expects that the object is in the "RefreshInProgress" state,
  283. // and that everything is in place and safe for a new rulebase to be loaded into the object. Once it's
  284. // done it will reset from the "RefreshInProgress" state and along the way will throw any errors that
  285. // are appropriate. The other functions can count on this one to polish off the various forms of rulebase
  286. // load activity.
  287. const cd::LogicCheck SaneRefreshProcessCheck("snf_RulebaseHandler::_snf_LoadNewRulebase():SaneRefreshProcessCheck(RefreshInProgress)");
  288. void snf_RulebaseHandler::_snf_LoadNewRulebase(){ // Common internal load/check routine.
  289. SaneRefreshProcessCheck(RefreshInProgress); // We only get called when this flag is set.
  290. try { MyCFGmgr.load(); } // Load a fresh copy of the configuration.
  291. catch(...) { // If something goes wrong:
  292. RefreshInProgress = false; // we are no longer "in refresh"
  293. throw ConfigurationError("_snf_LoadNewRulebase() MyCFGmgr.load() failed"); // throw the Configuration exception.
  294. }
  295. string RuleFilePath = MyCFGmgr.RuleFilePath(); // Get our rulebase file path and our
  296. string SecurityKey = MyCFGmgr.SecurityKey(); // security key from the CFG manager.
  297. if(0>=RuleFilePath.length()) { // If we don't have a path, we're hosed.
  298. RefreshInProgress = false; // We are no longer "in refresh"
  299. throw FileError("_snf_LoadNewRulebase() Zero length RuleFilePath"); // Can't load a RB file with no path!
  300. }
  301. if(0>=SecurityKey.length()) { // No security string? toast!
  302. RefreshInProgress = false; // We are no longer "in refresh"
  303. throw AuthenticationError("snf_LoadNewRulebase() Zero length SecurityKey"); // Can't authenticate without a key!
  304. }
  305. // Notify sub modules of the new configuration data.
  306. MyGeneration++; // Increment the generation number.
  307. snfCFGData& CFGData = (*(MyCFGmgr.ActiveConfiguration())); // Capture the active config...
  308. CFGData.Generation = MyGeneration; // Tag the configuration data.
  309. MyLOGmgr.configure(CFGData); // Update the LOGmgr's configuration.
  310. MyNETmgr.configure(CFGData); // Update the NETmgr's configuration.
  311. MyGBUdbmgr.configure(CFGData); // Update the GBUdbmgr's configuration.
  312. // Load the new rulebase locally (on stack) and see if it authenticates.
  313. TokenMatrix* TryThis = NULL; // We need our candidate to remain in scope.
  314. try { // This try block decodes the problem.
  315. try { // This try block does cleanup work.
  316. TryThis = new TokenMatrix(); // Grab a new Token Matrix
  317. TryThis->Load(RuleFilePath); // Load it from the provided file path
  318. TryThis->Validate(SecurityKey); // Validate it with the provided security key
  319. TryThis->Verify(SecurityKey); // Verify that it is not corrupt.
  320. }
  321. catch(...) { // Clean up after any exceptions.
  322. RefreshInProgress = false; // We're not refreshing now.
  323. if(TryThis) { // If we allocated a TokenMatrix then
  324. delete TryThis; // we need to reclaim the memory
  325. TryThis = 0; // and erase the pointer.
  326. } // With everything nice and clean we can
  327. throw; // rethrow he exception for decoding.
  328. }
  329. } // If nothing threw, we're golden!
  330. catch (const TokenMatrix::BadFile&) { // BadFile translates to FileError
  331. throw FileError("_snf_LoadNewRulebase() TokenMatrix::BadFile");
  332. }
  333. catch (const TokenMatrix::BadMatrix&) { // BadMatrix translates to AuthenticationError
  334. throw AuthenticationError("_snf_LoadNewRulebase() TokenMatrix::BadMatrix");
  335. }
  336. catch (const TokenMatrix::BadAllocation&) { // BadAllocation translates to AllocationError
  337. throw AllocationError("_snf_LoadNewRulebase() TokenMatrix::BadAllocation");
  338. }
  339. catch (const TokenMatrix::OutOfRange&) { // OutOfRange should never happen so PANIC!
  340. throw Panic("_snf_LoadNewRulebase() TokenMatrix::OutOfRange");
  341. }
  342. catch (...) { // Something unpredicted happens? PANIC!
  343. throw Panic("_snf_LoadNewRulebase() TokenMatrix.load() ???");
  344. }
  345. // At this point the rulebase looks good. If we need to go big-endian do it!
  346. #ifdef __BIG_ENDIAN__
  347. TryThis->FlipEndian(); // Flip tokens to big-endian format.
  348. #endif
  349. MyLOGmgr.updateActiveUTC(FileUTC(RuleFilePath)); // Update the Active Rulebase UTC.
  350. MyMutex.lock(); // Lock the mutex while changing state.
  351. OldRulebase = Rulebase; // Move the current rulebase and count to
  352. RetiringCount = CurrentCount; // the retiring slot.
  353. if(0>=RetiringCount && NULL!=OldRulebase) { // If nobody cares about the old rulebase
  354. delete OldRulebase; // then delete it, and wipe everything
  355. OldRulebase = NULL; // clean for the next retiree.
  356. RetiringCount = 0;
  357. }
  358. CurrentCount = 0; // Set the current count to zero (it's fresh!)
  359. Rulebase = TryThis; // Copy our new rulebase into production.
  360. MyMutex.unlock(); // Release the hounds!!!
  361. // If there is a GBUdb Ignore List, refresh with it (This might go elsewhere).
  362. // Failure to read the GBUdbIgnoreList if all else went well does not cause
  363. // the rulebase update (if any) to fail.
  364. /**** This section needs work ****/
  365. try {
  366. string IgnoreListPath = CFGData.paths_workspace_path;
  367. IgnoreListPath.append("GBUdbIgnoreList.txt");
  368. if(0 == MyGBUdb.readIgnoreList(IgnoreListPath.c_str())) // We must have at least 1 IP listed.
  369. throw ConfigurationError(
  370. "_snf_LoadNewRulebase() GBUdbIgnoreList min 1 entry!");
  371. }
  372. catch(...) { // Ignore list read might fail.
  373. RefreshInProgress = false; // If so, don't keep things hung.
  374. throw IgnoreListError("_snf_LoadNewRulebase() readIgnoreList() ???"); // If it does, throw FileError.
  375. }
  376. RefreshInProgress = false; // Done with the refresh process.
  377. return; // Our work is done here.
  378. }
  379. // open()
  380. // This loads a new rulebase (usually the first one only) into the handler. This is the first of two loading
  381. // methods on this object. This one checks for isBusy() because it is highly invasive. If it is called after
  382. // the object has been running it is important that it not run while anything in the object is active. This
  383. // is because it is likely in this case we would be loading an entirely new rulebase that would lead to odd
  384. // results if some scanner instances were activily using a different one.
  385. void snf_RulebaseHandler::open(const char* path, const char* licenseid, const char* authentication){
  386. MyMutex.lock(); // Lock the mutex while changing state.
  387. if(isBusy()) { // Be sure we're not busy.
  388. MyMutex.unlock(); throw Busy("snf_RulebaseHandler::open() busy"); // If we are then throw.
  389. }
  390. RefreshInProgress = true; // Set RefreshInProgress.
  391. MyMutex.unlock(); // Unlock the mutex and
  392. MyCFGmgr.initialize(path, licenseid, authentication); // Initialize our configuration.
  393. _snf_LoadNewRulebase(); // get on with loading the rulebase.
  394. MyGBUdbmgr.load(); // Load the GBUdb as configured.
  395. AutoRefresh(true); // Turn on Refresh by default.
  396. logThisInfo("--INITIALIZING--", 0, "Success"); // Log the happy event.
  397. return;
  398. }
  399. // refresh()
  400. // This loads a fresh copy of the current rulebase. This is the second loading method on the object. It is
  401. // specifically designed to work without stopping scanning activities. This one checks for isBusy() because
  402. // there may be an old rulebase that is not yet completely retired --- that is, some scanners may be using it.
  403. // If there is still an old rulebase on it's way out then we can't shove it aside without breaking something,
  404. // so we have to throw.
  405. //
  406. // Under normal circumstances, this call will cause a new rulebase to be loaded without disturbing any scans
  407. // underway on the current rulebase. The current rulebase will be put into retirement while any active scans
  408. // are completed, and then it will quietly go away when the last has finished. The new rulebase will take it's
  409. // place and will be handed out to all new grab() requests.
  410. void snf_RulebaseHandler::refresh(){ // Reloads the rulebase.
  411. MyMutex.lock(); // Lock the mutex while changing states.
  412. if(isBusy()) { // If we're busy then throw.
  413. MyMutex.unlock(); throw Busy("snf_RulebaseHandler::refresh() busy");
  414. }
  415. RefreshInProgress = true; // Set RefreshInProgress and
  416. MyMutex.unlock(); // unlock the mutex. Then get on with
  417. _snf_LoadNewRulebase(); // loading a fresh copy of the rulebase
  418. return;
  419. }
  420. void snf_RulebaseHandler::close(){ // Closes this handler.
  421. try {
  422. AutoRefresh(false); // Stop AutoRefresh if it's on.
  423. }
  424. catch(exception& e) { throw; } // Rethrow good exceptions.
  425. catch(...) { throw Panic("snf_RulebaseHandler::close() AutoRefresh(false) panic!"); } // Panic blank exceptions.
  426. try {
  427. MyXCImgr.stop(); // Stop the XCI manager.
  428. }
  429. catch(exception& e) { throw; } // Rethrow good exceptions.
  430. catch(...) { throw Panic("snf_RulebaseHandler::close() MyXCImgr.stop() panic!"); } // Panic blank exceptions.
  431. if(isBusy() || 0<CurrentCount || 0<ReferenceCount) { // Check that there is no activity.
  432. throw Busy("snf_RulebaseHandler::close() busy"); // With XCI stopped we should not
  433. } // be busy.
  434. try {
  435. MyLOGmgr.stop(); // Stop the LOG manager.
  436. }
  437. catch(exception& e) { throw; } // Rethrow good exceptions.
  438. catch(...) { throw Panic("snf_RulebaseHandler::close() MyLOGmgr.stop() panic!"); } // Panic blank exceptions.
  439. try {
  440. MyNETmgr.stop(); // Stop the NET manager.
  441. }
  442. catch(exception& e) { throw; } // Rethrow good exceptions.
  443. catch(...) { throw Panic("snf_RulebaseHandler::close() MyNETmgr.stop() panic!"); } // Panic blank exceptions.
  444. try {
  445. MyGBUdbmgr.stop(); // Stop the GBUdb manager.
  446. }
  447. catch(exception& e) { throw; } // Rethrow good exceptions.
  448. catch(...) { throw Panic("snf_RulebaseHandler::close() MyGBUdbmgr.stop() panic!"); } // Panic blank exceptions.
  449. try {
  450. if(NULL!=Rulebase) {delete Rulebase; Rulebase=NULL;} // If we have a Rulebase destroy it.
  451. }
  452. catch(exception& e) { throw; } // Rethrow good exceptions.
  453. catch(...) { throw Panic("snf_RulebaseHandler::close() delete Rulebase panic!"); } // Panic blank exceptions.
  454. try {
  455. if(NULL!=OldRulebase) {delete OldRulebase; OldRulebase=NULL;} // Shouldn't happen, but just in case.
  456. }
  457. catch(exception& e) { throw; } // Rethrow good exceptions.
  458. catch(...) { throw Panic("snf_RulebaseHandler::close() delete OldRulebase panic!"); } // Panic blank exceptions.
  459. }
  460. void snf_RulebaseHandler::use(){ // Make use of this Rulebase Handler.
  461. MyMutex.lock(); // Lock the object
  462. ReferenceCount++; // Boost the count
  463. MyMutex.unlock(); // Unlock the object
  464. }
  465. void snf_RulebaseHandler::unuse(){ // Finish with this Rulebase Handler.
  466. MyMutex.lock(); // Lock the object
  467. ReferenceCount--; // Reduce the count
  468. MyMutex.unlock(); // Unlock the object
  469. }
  470. // A word about Generation... In practice whenever the configuration or rulebase
  471. // changes the entire thing is reloaded. The Generation() function gives other
  472. // modules a way to know if they need to update their interpretation of the
  473. // configuration. They can keep track of the last Generation value they got and
  474. // compare it to the latest Generation. If the two are different then they need
  475. // to update their configuration - just in case it has changed.
  476. int snf_RulebaseHandler::Generation() { return MyGeneration; } // Returns the generation number.
  477. // A word about autopanics.
  478. // The first time throgh this we outsmarted ourselves with an ellaborate
  479. // wait-to-insert scheme. That led to the possibilty of a deadlock. Now we
  480. // copy the (usually empty or very short) set of rule panics to the
  481. // configuration packet when it is grabbed and only use the one mutext to hold
  482. // the configuration steady while doing so. All queries are made to the local
  483. // copy of the panic list and all writes are made, under mutex, to the active
  484. // configuration. Simpler, no significant penalty, and no more deadlocks.
  485. // A word about configuration packets.
  486. // Along the way we simplified things by making the snfCFGPacket do it's own
  487. // grab and drop upon construction and destruction. This way we don't have to
  488. // remember to handle all possible cases during a scan or other opertion -- once
  489. // the operation goes out of scope the configuration packet drop()s with it.
  490. void snf_RulebaseHandler::grab(snfCFGPacket& CP) { // Activate this Rulebase.
  491. ScopeMutex HoldStillPlease(MyMutex); // Lock the rulebase until we're done.
  492. CurrentCount++; // Boost the count for myself.
  493. CP.MyTokenMatrix = Rulebase; // Grab the current rulebase.
  494. CP.MyCFGData = MyCFGmgr.ActiveConfiguration(); // Grab the active configuration.
  495. CP.RulePanics = MyCFGmgr.ActiveConfiguration()->RulePanicHandler.IntegerSet; // Copy the RulePanic set.
  496. }
  497. void snf_RulebaseHandler::drop(snfCFGPacket& CP) { // Deactiveate this Rulebase.
  498. const TokenMatrix* t = CP.MyTokenMatrix; // Grab the token matrix pointer.
  499. CP.MyCFGData = NULL; // Null the configuration pointer.
  500. ScopeMutex HoldStillPlease(MyMutex); // Lock the rulebase until we're done.
  501. if(t==Rulebase) { // If we're dropping the current rulebase
  502. CurrentCount--; // then reduce the current count.
  503. } else // If not that then...
  504. if(t==OldRulebase) { // If we're dropping the old rulebase
  505. RetiringCount--; // reduce the retiring count and check...
  506. if(0>=RetiringCount) { // to see if it is completely retired.
  507. if(NULL!=OldRulebase) delete OldRulebase; // If it is then delete it and
  508. OldRulebase = NULL; RetiringCount = 0; // reset it's pointer and counter.
  509. }
  510. } else { // If we're dropping something else,
  511. throw Panic("snf_RulebaseHandler::drop() panic"); // it is time to panic, so, then PANIC!
  512. }
  513. }
  514. // When adding a rule panic entry the rulebase and configuration state cannot
  515. // be changed, nor grabbed by an snfCFGPacket. This ensures that the IntegerSet
  516. // is only adjusted by one thread at a time and that any threads using the set
  517. // will have a consistent result based on their last grab().
  518. void snf_RulebaseHandler::addRulePanic(int RuleID) { // Add a rule panic id dynamically.
  519. ScopeMutex JustMe(MyMutex); // Freeze the rulebase while we adjust
  520. MyCFGmgr.ActiveConfiguration() // the active configuration to
  521. ->RulePanicHandler.IntegerSet.insert(RuleID); // insert the new rule panic ruleid.
  522. } // When we're done, unlock and move on.
  523. IPTestRecord& snf_RulebaseHandler::performIPTest(IPTestRecord& I) { // Perform an IP test.
  524. snfCFGPacket MyCFGPacket(this); // We need a CFG packet.
  525. try { // Safely process the IP.
  526. if(false == MyCFGPacket.bad()) { // If we've got a good packet:
  527. I.G = MyGBUdb.getRecord(I.IP); // Lookup the IP in GBUdb.
  528. I.R = MyCFGPacket.Config()->RangeEvaluation(I.G); // Evaluate it's statistics.
  529. // Convert the RangeEvaluation into the configured Code
  530. switch(I.R) {
  531. case Unknown: // Unknown - not defined.
  532. case Normal: // Benefit of the doubt.
  533. case New: { // It is new to us.
  534. I.Code = 0; // Zero is the default - no code.
  535. break;
  536. }
  537. case White: { // This is a good guy.
  538. I.Code = MyCFGPacket.Config()->WhiteRangeHandler.Symbol;
  539. break;
  540. }
  541. case Caution: { // This is suspicious.
  542. I.Code = MyCFGPacket.Config()->CautionRangeHandler.Symbol;
  543. break;
  544. }
  545. case Black: { // This is bad.
  546. I.Code = MyCFGPacket.Config()->BlackRangeHandler.Symbol;
  547. break;
  548. }
  549. case Truncate: { // Don't even bother looking.
  550. I.Code = MyCFGPacket.Config()
  551. ->gbudb_regions_black_truncate_symbol;
  552. break;
  553. }
  554. }
  555. } // If something is broken we punt.
  556. } catch (...) {} // Ignore exceptions (none expected)
  557. return I; // Return the processed record.
  558. }
  559. void snf_RulebaseHandler::logThisIPTest(IPTestRecord& I, string Action) { // Log an IP test result & action.
  560. MyLOGmgr.logThisIPTest(I, Action);
  561. }
  562. void snf_RulebaseHandler::logThisError( // Log an error message.
  563. string ContextName, int Code, string Text
  564. ) {
  565. MyLOGmgr.logThisError(ContextName, Code, Text);
  566. }
  567. void snf_RulebaseHandler::logThisInfo( // Log an informational message.
  568. string ContextName, int Code, string Text
  569. ) {
  570. MyLOGmgr.logThisInfo(ContextName, Code, Text);
  571. }
  572. string snf_RulebaseHandler::PlatformVersion(string NewPlatformVersion) { // Set platform version info.
  573. return MyLOGmgr.PlatformVersion(NewPlatformVersion);
  574. }
  575. string snf_RulebaseHandler::PlatformVersion() { // Get platform version info.
  576. return MyLOGmgr.PlatformVersion();
  577. }
  578. string snf_RulebaseHandler::PlatformConfiguration() { // Get platform configuration.
  579. ScopeMutex LockAndGrab(MyMutex); // Freeze things for a moment and
  580. return MyCFGmgr.ActiveConfiguration()->PlatformElementContents; // copy the platform configuration.
  581. }
  582. string snf_RulebaseHandler::EngineVersion() { // Get engine version info.
  583. return MyLOGmgr.EngineVersion();
  584. }
  585. void snf_RulebaseHandler::
  586. XCIServerCommandHandler(snfXCIServerCommandHandler& XCH) { // Registers a new XCI Srvr Cmd handler.
  587. ScopeMutex ThereCanBeOnlyOne(XCIServerCommandMutex); // Serialize access to this resource.
  588. myXCIServerCommandHandler = &XCH; // Assign the new handler as provided.
  589. }
  590. string snf_RulebaseHandler::processXCIServerCommandRequest(snf_xci& X) { // Handle a parsed XCI Srvr Cmd request.
  591. ScopeMutex ThereCanBeOnlyOne(XCIServerCommandMutex); // Serialize access to this resource.
  592. if(0 == myXCIServerCommandHandler) { // If we don't have a handler then
  593. snfXCIServerCommandHandler H; // create a base handler and
  594. return H.processXCIRequest(X); // return it's default response.
  595. } // If we do have a handler then pass
  596. return myXCIServerCommandHandler->processXCIRequest(X); // on the request and return the
  597. } // response.
  598. //// snf_IPTestEngine Methods
  599. snf_IPTestEngine::snf_IPTestEngine() : // The constructor is simple - it
  600. Lookup(NULL), ScanData(NULL) { // sets up our internal references.
  601. } // Before use these must be set.
  602. void snf_IPTestEngine::setGBUdb(GBUdb& G) { // Here's how we set the GBUdb.
  603. Lookup = &G;
  604. }
  605. void snf_IPTestEngine::setScanData(snfScanData& S) { // Here's how we set the ScanData object.
  606. ScanData = &S;
  607. }
  608. void snf_IPTestEngine::setCFGData(snfCFGData& C) { // Here's how we set the CFGData.
  609. CFGData = &C;
  610. }
  611. void snf_IPTestEngine::setLOGmgr(snfLOGmgr& L) { // Here's how we set the LOGmgr.
  612. LOGmgr = &L;
  613. }
  614. // 20090127 _M Added special handling for forced IP sources. First, they are
  615. // always considered the source and second if they are in the GBUdb ignore list
  616. // then GBUdb training bypass is established.
  617. string& snf_IPTestEngine::test(string& input, string& output) { // Perform IP lookups and put IPs into ScanData.
  618. if(NULL == Lookup || NULL == ScanData) { // If we are not set up properly then we
  619. output = "{IPTest Config Error}"; // will return an error string.
  620. return output;
  621. }
  622. try { // If we're out of IP records, no analysis.
  623. IPScanRecord& I = ScanData->newIPScanRecord(); // Grab a new IP scan record and
  624. IP4Address IP = input; // Convert the string to an IP.
  625. // Identify forced Source IP addresses
  626. bool ThisSourceIsForced = ( // This IP is a forced source IP if:
  627. (0 == I.Ordinal) && ( // we are looking at the first IP and
  628. (0UL != ScanData->CallerForcedSourceIP()) || // either the Caller forced the IP or
  629. (0UL != ScanData->HeaderDirectiveSourceIP()) // the IP was forced by a header directive.
  630. )
  631. );
  632. // Bad IPs are possible, especially if the source was forced. In that
  633. // case forced source IP is meaningless so we want to ignore it and
  634. // we want to make the case visible in the logs. An ordinary IP that
  635. // is invalid has no consequence so we simply skip those.
  636. // Note that a source IP that has it's ignore flag set causes an
  637. // implied training bypass inside the scan function. Setting the bad
  638. // IP as the source and setting it's ignore flag will have the desired
  639. // effect.
  640. if(0UL == IP) { // If we got a 0 or a bad conversion then
  641. output = "{0.0.0.0 Is Not A Usable IP}"; // we won't be testing this IP.
  642. if(ThisSourceIsForced) { // If this ip is a forced source then
  643. I.GBUdbData.Flag(Ignore); // we will force a training bypass,
  644. ScanData->SourceIPRecord(I); // we will record it as the source,
  645. ScanData->SourceIPEvaluation = output; // and capture the error output.
  646. }
  647. return output;
  648. }
  649. if(0xFFFFFFFF == IP) { // If we got a 255.255.255.255 then
  650. output = "{255.255.255.255 Is Not A Usable IP}"; // we won't be testing this IP.
  651. if(ThisSourceIsForced) { // If this ip is a forced source then
  652. I.GBUdbData.Flag(Ignore); // we will force a training bypass,
  653. ScanData->SourceIPRecord(I); // we will record it as the source,
  654. ScanData->SourceIPEvaluation = output; // and capture the error output.
  655. }
  656. return output;
  657. }
  658. GBUdbRecord R = Lookup->getRecord(IP); // Get the GBUdb record for it.
  659. I.IP = IP; // store the IP and the
  660. I.GBUdbData = R; // GBUdb record we retrieved.
  661. output = "{"; // Next we start to build our IP data insert.
  662. ostringstream S; // We will use a string stream for formatting.
  663. switch(R.Flag()) { // Identify the flag data for this IP.
  664. case Good: S << "Good "; break;
  665. case Bad: S << "Bad "; break;
  666. case Ugly: S << "Ugly "; break;
  667. case Ignore: S << "Ignore "; break;
  668. }
  669. S << "c=" << R.Confidence() << " " // Include the Confidence and
  670. << "p=" << R.Probability(); // Probability.
  671. // Process ordinary Source IP addresses
  672. if( // The message source IP address is the
  673. (false == ScanData->FoundSourceIP()) && // first IP we find that is either forced
  674. (ThisSourceIsForced || (Ignore != R.Flag())) // OR is NOT part of our infrastructure.
  675. ) { // When we find the correct source IP:
  676. if( // Check to see if we're drilling down.
  677. (false == ThisSourceIsForced) && // We drill when the source is NOT forced
  678. (ScanData->isDrillDownSource(I)) // AND we have a matching drilldown.
  679. ) {
  680. Lookup->setIgnore(IP); // If we're drilling down ignore this IP.
  681. }
  682. else { // If not drilling down this is the source:
  683. ScanData->SourceIPRecord(I); // we log it in as the source
  684. S << " Source"; // and report our findings in our tag.
  685. // Since we are dealing with our source IP
  686. // this is a good place to evaluate our truncate feature.
  687. snfIPRange IPR =
  688. ScanData->SourceIPRange(CFGData->RangeEvaluation(R)); // Establish the IP range for this scan.
  689. // We will also emit a range identifier for pattern matches that might use it.
  690. switch(IPR) {
  691. case Unknown: { S << " Unknown"; break; } // Unknown - not defined.
  692. case White: { S << " White"; break; } // This is a good guy.
  693. case Normal: { S << " Normal"; break; } // Benefit of the doubt.
  694. case New: { S << " New"; break; } // It is new to us.
  695. case Caution: { S << " Caution"; break; } // This is suspicious.
  696. case Black: { S << " Black"; break; } // This is bad.
  697. case Truncate: { S << " Truncate"; break; } // Don't even bother looking.
  698. }
  699. ScanData->SourceIPEvaluation = S.str(); // Capture the source IP eval.
  700. // The RangeEvaluation() call above settles a lot of questions for us.
  701. // The Truncate return code only happens when the IP is either Bad w/
  702. // truncate turned on, or the statistics place the IP in the Truncate
  703. // range. If the Good flag is set the function always returns White so
  704. // here we only have to check for the Truncate flag.
  705. if(Truncate == IPR) { // If all of the conditions are met
  706. ScanData->GBUdbTruncateTriggered = true; // then Truncate has been triggered.
  707. ScanData->GBUdbPeekTriggered = LOGmgr->OkToPeek( // Since truncate was triggered, see if
  708. CFGData->gbudb_regions_black_truncate_peek_one_in); // we would also trigger a peek.
  709. // The reason we check the truncate on_off flag here is that the
  710. // IP range _may_ return a Truncate result if no Flags are set on
  711. // the IP and the IP is far enough into the black to reach the
  712. // Truncate threshold.
  713. if(CFGData->gbudb_regions_black_truncate_on_off) { // If truncate is on either peek or truncate.
  714. if(ScanData->GBUdbPeekTriggered) { // If a peek has been triggered then
  715. ScanData->GBUdbPeekExecuted = true; // mark the event and don't truncate.
  716. } else { // If a peek was not triggered then
  717. ScanData->GBUdbTruncateExecuted = true; // Record our trucnate action.
  718. output = ""; // Set up the truncate signal (empty string)
  719. return output; // and return it! We're done!
  720. }
  721. }
  722. }
  723. }
  724. }
  725. // If we're not truncating then we're going to return our IP evaulation tag
  726. // to the filter chain function module so it can emit it into the stream.
  727. output.append(S.str());
  728. output.append("}");
  729. }
  730. catch(snfScanData::NoFreeIPScanRecords) {
  731. output = "{too_many}";
  732. }
  733. catch(...) {
  734. output = "{fault}";
  735. }
  736. return output;
  737. }
  738. //// Engine Handler Methods
  739. snf_EngineHandler::~snf_EngineHandler(){ // Shutdown clenas up and checks for safety.
  740. if(isReady()) close(); // If we're live, close on our way out.
  741. }
  742. void snf_EngineHandler::open(snf_RulebaseHandler* Handler){ // Light up the engine.
  743. MyMutex.lock(); // Serialize this...
  744. if(isReady()) { // If we're already open then we need to
  745. MyMutex.unlock(); // unlock this object and let them know
  746. throw Busy("snf_EngineHandler::open() busy"); // we are busy.
  747. } // If we're not busy, then let's light it up.
  748. MyRulebase=Handler; // Install our rulebase handler.
  749. MyRulebase->use(); // Up the use count to let it know we're here.
  750. MyIPTestEngine.setGBUdb(MyRulebase->MyGBUdb); // Set up the IPTester's GBUdb.
  751. MyIPTestEngine.setScanData(MyScanData); // Set up the IPTester's ScanData reference.
  752. MyIPTestEngine.setLOGmgr(MyRulebase->MyLOGmgr); // Set up the IPTester's LOGmgr.
  753. MyMutex.unlock(); // Unlock our mutex, then...
  754. return; // our work is done.
  755. }
  756. bool snf_EngineHandler::isReady(){ // Is the Engine good to go?
  757. return (NULL!=MyRulebase); // Have rulebase will travel.
  758. }
  759. void snf_EngineHandler::close(){ // Close down the engine.
  760. MyMutex.lock(); // Serialize this...
  761. if(!isReady()){ // If we're not already open we can't close.
  762. MyMutex.unlock(); // Something is seriously wrong, so unlock
  763. throw Panic("snf_EngineHandler::close() !isReady panic"); // and hit the panic button!
  764. } // But, if everything is ok then we can
  765. MyRulebase->unuse(); // unuse our rulebase and quietly forget
  766. MyRulebase = NULL; // about it.
  767. if(NULL!=CurrentMatrix) { // If we have a leftover evaluation matrix
  768. delete CurrentMatrix; // we can let that go and forget about
  769. CurrentMatrix = NULL; // it as well.
  770. }
  771. MyMutex.unlock(); // Finally, we unlock our mutex and...
  772. return; // Our work is done here.
  773. }
  774. enum PatternResultTypes { // To train GBUdb we need a generalized
  775. NoPattern, // way to evaluate the results from the
  776. WhitePattern, // snf pattern matching scan.
  777. BlackPattern,
  778. IPPattern,
  779. AboveBandPattern
  780. };
  781. // In order to optimize message file reads when header injection is not activated
  782. // we need to look ahead to see if header injection is likely to be turned on when
  783. // we do the scan. This is a short term fix. The better fix might be to perform
  784. // the configuration load prior to scanning the message -- but that is a much larger
  785. // refactoring that ties up configuration and rulebase resources for a longer time.
  786. // Instead we're going to take an optimistic route and just peek at the configuration.
  787. // If the configuration changes while we're loading the file to be scanned then
  788. // we have two cases. If we go from XHDRInject off to XHDRInject on then we will
  789. // miss adding headers to the message - not a bad outcome. If we go from XHDRInject
  790. // on to XHDRInject off then we might emit headers for an extra message - also not
  791. // a bad outcome.
  792. bool snf_RulebaseHandler::testXHDRInjectOn() {
  793. ScopeMutex HoldStillPlease(MyMutex); // Lock the rulebase until we're done.
  794. snfCFGData* myCFG = MyCFGmgr.ActiveConfiguration(); // Grab the active configuration.
  795. bool myXHDRInjectOnFlag = (LogOutputMode_Inject == myCFG->XHDROutput_Mode); // True if output mode is inject.
  796. return myXHDRInjectOnFlag; // return the result.
  797. }
  798. int snf_EngineHandler::scanMessageFile( // Scan this message file.
  799. const string MessageFilePath, // -- this is the file path (and id)
  800. const int MessageSetupTime, // -- setup time already used.
  801. const IP4Address MessageSource // -- message source IP (for injection).
  802. ) {
  803. Timer AdditionalSetupTime;
  804. ScopeMutex DoingAFileScan(FileScan); // Protect MyScanData @ this entry.
  805. // Preliminary setup. Clearing the ScanData resets the ReadyToClear flag
  806. // and allows us to set some data for more accurate tracking and so that if
  807. // something goes wrong the ScanData will be helpful in determining the
  808. // state of the engine.
  809. MyScanData.clear(); // Clear the scan data.
  810. MyScanData.StartOfJobUTC = MyRulebase->MyLOGmgr.Timestamp(); // Set the job start timestamp.
  811. MyScanData.ScanName = MessageFilePath;
  812. // Now that the preliminaries are established we can begin our work.
  813. int MessageFileSize = 0; // Here will be the size of it.
  814. ifstream MessageFile; // Here will be our input file.
  815. MessageFile.exceptions( // It will throw exceptions for
  816. ifstream::eofbit | ifstream::failbit | ifstream::badbit // these unwanted events.
  817. );
  818. try { // Try opening the message file.
  819. MessageFile.open(MessageFilePath.c_str(), ios::in | ios::binary); // Open the file, binary mode.
  820. MessageFile.seekg(0, ios::end); // Find the end of the file,
  821. MessageFileSize = MessageFile.tellg(); // read that position as the size,
  822. MessageFile.seekg(0, ios::beg); // then go back to the beginning.
  823. MyScanData.ScanSize = MessageFileSize; // Capture the message file size.
  824. }
  825. catch(...) { // Trouble? Throw FileError.
  826. MyRulebase->MyLOGmgr.logThisError( // Log the error.
  827. MyScanData, "scanMessageFile().open",
  828. snf_ERROR_MSG_FILE, "ERROR_MSG_FILE"
  829. );
  830. throw FileError("snf_EngineHandler::scanMessageFile() Open/Seek");
  831. }
  832. if(0 >= MessageFileSize) { // Handle zero length files.
  833. MessageFile.close(); // No need to keep this open.
  834. MyRulebase->MyLOGmgr.logThisError( // Log the error.
  835. MyScanData, "scanMessageFile().isFileEmpty?",
  836. snf_ERROR_MSG_FILE, "ERROR_MSG_FILE"
  837. );
  838. throw FileError("snf_EngineHandler::scanMessageFile() FileEmpty!");
  839. }
  840. bool isXHeaderInjectionOn = MyRulebase->testXHDRInjectOn();
  841. bool noNeedToReadFullFile = (false == isXHeaderInjectionOn);
  842. if(noNeedToReadFullFile) {
  843. MessageFileSize = min(MessageFileSize, snf_ScanHorizon);
  844. }
  845. vector<unsigned char> MessageBuffer; // Allocate a buffer and size
  846. try { MessageBuffer.resize(MessageFileSize, 0); } // it to fit the message.
  847. catch(...) { // Trouble? Throw AllocationError.
  848. MyRulebase->MyLOGmgr.logThisError( // Log the error.
  849. MyScanData, "scanMessageFile().alloc",
  850. snf_ERROR_MSG_FILE, "ERROR_MSG_ALLOC"
  851. );
  852. throw AllocationError("snf_EngineHandler::scanMessageFile() Alloc");
  853. }
  854. try { MessageFile.read((char*) &MessageBuffer[0], MessageFileSize); } // Read the file into the buffer.
  855. catch(...) {
  856. MyRulebase->MyLOGmgr.logThisError( // Log the error.
  857. MyScanData, "scanMessageFile().read",
  858. snf_ERROR_MSG_FILE, "ERROR_MSG_READ"
  859. );
  860. throw FileError("snf_EngineHandler::scanMessageFile() Read");
  861. }
  862. MessageFile.close(); // Close the file.
  863. // Additional Setup Time will be captured as the call is made.
  864. int ScanResultCode = scanMessage( // Scan the message we've loaded.
  865. &MessageBuffer[0], // Here is the buffer pointer,
  866. MessageBuffer.size(), // here is the size of the message,
  867. MessageFilePath, // the path is the identifier,
  868. (AdditionalSetupTime.getElapsedTime() + MessageSetupTime), // and this is our setup time total.
  869. MessageSource // Pass on the source if provided.
  870. );
  871. // Inject headers if required.
  872. if(isXHeaderInjectionOn) { // If we are to inject headers:
  873. const char* XHDRInjStage = "Begin"; // Keep track of what we're doing.
  874. try {
  875. // The insertion point will be at the end of the existing headers.
  876. // We pick that point to be right between the two <cr><lf> so that
  877. // the first blank line will appear at the end of our headers.
  878. // We accommodate either <cr><lf> or <lf> line endings.
  879. // We are careful not to search past the end of unreasonably short
  880. // message files.
  881. unsigned int InsertPoint = 0; // Find the insertion point.
  882. bool UseLFOnly = false; // Use \n line endings in files?
  883. bool CRLFPresent = false; // Detected \r\n pairs?
  884. unsigned int BiggestPatternSize = 4; // How far we look ahead.
  885. bool BigEnoughMessage = BiggestPatternSize < MessageBuffer.size();
  886. if(BigEnoughMessage){
  887. unsigned int Limit = MessageBuffer.size() - BiggestPatternSize;
  888. bool DataWasSkipped = MessageBuffer.size() > MyScanData.ScanSize;
  889. unsigned int i = 0;
  890. if(DataWasSkipped) { // If our scanner skipped data at
  891. i = MessageBuffer.size() - MyScanData.ScanSize; // the top of the message buffer then
  892. } // we will skip it too.
  893. for(; i < Limit; i++) { // Search for the first blank line.
  894. if( // Detect CRLF pairs if present.
  895. false == CRLFPresent &&
  896. '\r' == MessageBuffer.at(i) &&
  897. '\n' == MessageBuffer.at(i + 1)
  898. ) CRLFPresent = true;
  899. if( // In a properly formatted RFC822
  900. '\r' == MessageBuffer.at(i) && // message that looks like
  901. '\n' == MessageBuffer.at(i + 1) && // <cr><lf><cr><lf>
  902. '\r' == MessageBuffer.at(i + 2) &&
  903. '\n' == MessageBuffer.at(i + 3)
  904. ) {
  905. InsertPoint = i + 2;
  906. break;
  907. } else
  908. if( // In some bizarre cases it might
  909. '\n' == MessageBuffer.at(i) && // look like <lf><lf>.
  910. '\n' == MessageBuffer.at(i + 1)
  911. ) {
  912. InsertPoint = i + 1;
  913. UseLFOnly = true; // We have to strip <CR> from our
  914. break; // injected header line ends.
  915. }
  916. }
  917. }
  918. // Here we must interpret the results of our search. Do we know where
  919. // our insert point is or do we punt and use the top of the message?
  920. if(0 == InsertPoint) { // No blank line? We need to punt.
  921. if(false == CRLFPresent) { // What kind of line ends do we use?
  922. UseLFOnly = true; // If no CRLF found use LF only.
  923. } // Either way we will be inserting
  924. } // our headers at the top of the msg.
  925. // At this point we know where to split the message and insert
  926. // our X Headers.
  927. XHDRInjStage = "Open Temp File"; // Update our process monitor.
  928. string TempFileName = MessageFilePath; // Prepare a temp file name
  929. TempFileName.append(".tmp"); // based on the message file.
  930. ofstream TempFile; // Here will be our temp file.
  931. TempFile.exceptions(ofstream::failbit | ofstream::badbit); // It will throw these exceptions.
  932. TempFile.open(TempFileName.c_str(), ios::binary | ios::trunc); // Open and truncate the file.
  933. // If our insert point is the top of the message we'll skip this.
  934. if(0 < InsertPoint) { // If we have an insert point:
  935. XHDRInjStage = "Write Temp File.1"; // Update our process monitor.
  936. TempFile.write( // Write the message file up
  937. reinterpret_cast<char*>(&MessageBuffer[0]), // to our split.
  938. InsertPoint
  939. );
  940. }
  941. // If our file has \n line ends we need to strip the \r from our
  942. // rfc822 \r\n line ends.
  943. XHDRInjStage = "XHDR <CR><LF> to <LF>";
  944. if(true == UseLFOnly) { // If we are using <LF> only:
  945. string ReworkedHeaders = ""; // Make a new string and rework
  946. for( // our headers.
  947. string::iterator iS = MyScanData.XHDRsText.begin(); // Run through the headers one
  948. iS != MyScanData.XHDRsText.end(); iS++ // byte at a time.
  949. ) {
  950. if('\r' != (*iS)) ReworkedHeaders.push_back(*iS); // Strip out any <CR> chars.
  951. }
  952. MyScanData.XHDRsText.swap(ReworkedHeaders); // Swap in our reworked headers.
  953. }
  954. // Now we are ready to inject our headers.
  955. XHDRInjStage = "Write Temp File.2"; // Update our process monitor.
  956. TempFile.write( // Inject our headers.
  957. MyScanData.XHDRsText.c_str(),
  958. MyScanData.XHDRsText.length()
  959. );
  960. XHDRInjStage = "Write Temp File.3"; // Update our process monitor.
  961. TempFile.write( // Write the rest of the message.
  962. reinterpret_cast<char*>(&MessageBuffer[InsertPoint]),
  963. MessageBuffer.size() - InsertPoint
  964. );
  965. XHDRInjStage = "Close Temp File"; // Update our process monitor.
  966. TempFile.close(); // Close the file (flushing it).
  967. Sleeper PauseBeforeRetry(300); // Delay to use between retries.
  968. XHDRInjStage = "Drop Msg"; // Update our process monitor.
  969. if(remove(MessageFilePath.c_str())) { // Remove the old message file
  970. PauseBeforeRetry(); // If it fails, pause and retry.
  971. if(remove(MessageFilePath.c_str())) { // If that fails,
  972. PauseBeforeRetry(); // pause, then try once more.
  973. if(remove(MessageFilePath.c_str())) { // If that fails, throw.
  974. throw XHDRError("XHDR injector can't remove original!");
  975. }
  976. }
  977. }
  978. XHDRInjStage = "Rename Temp -> Msg"; // Update our process monitor.
  979. if(rename(TempFileName.c_str(), MessageFilePath.c_str())) { // Make Temp our new message file.
  980. PauseBeforeRetry(); // If it fails, pause and retry.
  981. if(rename(TempFileName.c_str(), MessageFilePath.c_str())) { // If that fails,
  982. PauseBeforeRetry(); // pause then try once more.
  983. if(rename(TempFileName.c_str(), MessageFilePath.c_str())) { // If that fails, throw.
  984. throw XHDRError("XHDR injector can't rename tmp file!");
  985. }
  986. }
  987. }
  988. }
  989. catch(XHDRError& e) { // For full XHDRError exceptions.
  990. string ERROR_MSG_XHDRi = "ERROR_MSG_XHDRi: "; // Format the XHDRInj error msg.
  991. ERROR_MSG_XHDRi.append(XHDRInjStage);
  992. ERROR_MSG_XHDRi.append(" ");
  993. ERROR_MSG_XHDRi.append(e.what());
  994. MyRulebase->MyLOGmgr.logThisError( // Log the error.
  995. MyScanData, "scanMessageFile().xhdr.inject",
  996. snf_ERROR_MSG_FILE, ERROR_MSG_XHDRi
  997. );
  998. throw; // Rethrow any XHDRError exceptions.
  999. }
  1000. catch(exception& e) { // For ordinary runtime exceptions.
  1001. string ERROR_MSG_XHDRi = "ERROR_MSG_XHDRi: "; // Format the XHDRInj error msg.
  1002. ERROR_MSG_XHDRi.append(XHDRInjStage);
  1003. ERROR_MSG_XHDRi.append(" ");
  1004. ERROR_MSG_XHDRi.append(e.what());
  1005. MyRulebase->MyLOGmgr.logThisError( // Log the error.
  1006. MyScanData, "scanMessageFile().xhdr.inject",
  1007. snf_ERROR_MSG_FILE, ERROR_MSG_XHDRi
  1008. );
  1009. throw XHDRError(ERROR_MSG_XHDRi); // Rethrow as XHDRError exceptions.
  1010. }
  1011. catch(...) { // If we encounter a problem then
  1012. string ERROR_MSG_XHDRi = "ERROR_MSG_XHDRi: "; // Format the XHDRInj error msg.
  1013. ERROR_MSG_XHDRi.append(XHDRInjStage);
  1014. MyRulebase->MyLOGmgr.logThisError( // Log the error.
  1015. MyScanData, "scanMessageFile().xhdr.inject",
  1016. snf_ERROR_MSG_FILE, ERROR_MSG_XHDRi
  1017. );
  1018. string XHDRError_msg = "Message Rewrite Failed: "; // Format our throw message with
  1019. XHDRError_msg.append(XHDRInjStage); // our detailed stage data and
  1020. throw XHDRError(XHDRError_msg); // throw our special exception.
  1021. }
  1022. }
  1023. // Create an .xhdr file if required.
  1024. if(MyScanData.XHeaderFileOn) {
  1025. try {
  1026. ofstream XHDRFile; // Output file will be XHDRFile.
  1027. XHDRFile.exceptions(ofstream::failbit | ofstream::badbit); // These events will throw exceptions.
  1028. string XHDRFileName = MessageFilePath; // Build the XHDR file name by adding
  1029. XHDRFileName.append(".xhdr"); // .xhdr to the message file name.
  1030. XHDRFile.open(XHDRFileName.c_str(), ios::binary | ios::trunc); // Open (and truncate) the file.
  1031. XHDRFile << MyScanData.XHDRsText; // Spit out the XHDRs.
  1032. XHDRFile.close(); // All done.
  1033. }
  1034. catch(...) { // If we encounter a problem then
  1035. MyRulebase->MyLOGmgr.logThisError( // Log the error.
  1036. MyScanData, "scanMessageFile().xhdr.file",
  1037. snf_ERROR_MSG_FILE, "ERROR_MSG_XHDRf"
  1038. );
  1039. throw XHDRError(".xhdr file write failed"); // throw our special exception.
  1040. }
  1041. }
  1042. return ScanResultCode; // Return the actual result, of course.
  1043. }
  1044. string snf_EngineHandler::extractMessageID( // Find and return the first Message-ID
  1045. const unsigned char* Msg, // Input the Message buffer to search
  1046. const int Len // and the length of the buffer.
  1047. ) {
  1048. string ExtractedID = ""; // Start with an empty string.
  1049. bool FoundID = false; // Haven't found it yet.
  1050. int C = 0; // Cursor position.
  1051. while(!FoundID && (C < (Len - 12))) { // Loop through the Msg looking for
  1052. if( // the Message-ID: header.
  1053. ('\n' == Msg[C]) && // Starting at the new line find
  1054. ('M' == Msg[C + 1] || 'm' == Msg[C + 1]) && // Message-ID: (per RFC822)
  1055. ('e' == Msg[C + 2] || 'E' == Msg[C + 2]) &&
  1056. ('s' == Msg[C + 3] || 'S' == Msg[C + 3]) && // We use an unrolled comparison
  1057. ('s' == Msg[C + 4] || 'S' == Msg[C + 4]) && // loop here for raw speed and
  1058. ('a' == Msg[C + 5] || 'A' == Msg[C + 5]) && // optimization. Note that we
  1059. ('g' == Msg[C + 6] || 'G' == Msg[C + 6]) && // compare the most likely characters
  1060. ('e' == Msg[C + 7] || 'E' == Msg[C + 7]) && // first in each case, and we don't
  1061. ('-' == Msg[C + 8]) && // need to go through a buffer length
  1062. ('I' == Msg[C + 9] || 'i' == Msg[C + 9]) && // check at each byte for partial
  1063. ('D' == Msg[C + 10] || 'd' == Msg[C + 10]) && // matches.
  1064. (':' == Msg[C + 11]) &&
  1065. (' ' == Msg[C + 12] || '\t' == Msg[C + 12])
  1066. ) {
  1067. C = C + 13; // Starting just after the space
  1068. while(C < Len) { // and staying within bounds
  1069. unsigned char X = Msg[C]; // grab each character in the ID.
  1070. if(isprint(X)) { // If it is printable,
  1071. if(' ' == X) X = '_'; // massage out the spaces as _ and
  1072. if(127 < X) X = '|'; // high characters as | and
  1073. if('\'' == X || '\"' == X) X = '`'; // ' or " to ` in order to make the
  1074. ExtractedID.push_back(X); // ID safe for logging, then push
  1075. } else // the result into our string. When
  1076. if('\r' == X || '\n' == X) break; /* leave copy loop */ // we reach the end we're done.
  1077. ++C; // else get ready for the next byte.
  1078. }
  1079. FoundID = true; // Set the flag: we found Message-ID:
  1080. break; /* leave search loop */ // We got what we came for. Break!
  1081. } else { // When we don't find the Message-ID:
  1082. if( // we check for end of headers.
  1083. ('\n' == Msg[C] && '\n' == Msg[C+1]) || // Either <LF><LF> or
  1084. ('\r' == Msg[C] && '\n' == Msg[C+1] && // <CR><LF><CF><LF>
  1085. '\r' == Msg[C+2] && '\n' == Msg[C+3])
  1086. ) { // If we've found the end of headers
  1087. break; // we're done looking. If we did not
  1088. } // find the end of headers then
  1089. ++C; // we move to the next position.
  1090. }
  1091. }
  1092. // At this point we either have the Extracted ID, or we need a substitute.
  1093. if(0 == ExtractedID.length()) { // If we need a substitute ID then
  1094. MyRulebase->MyLOGmgr.SerialNumber(ExtractedID); // use the next available serial number.
  1095. }
  1096. return ExtractedID; // Return the extracted id or substitute.
  1097. }
  1098. const cd::LogicFault FaultBadMessageBuffer1("snf_EngineHandler::scanMessage():FaultBadMessageBuffer1(NULL == inputMessageBuffer)");
  1099. const cd::LogicFault FaultBadMessageBuffer2("snf_EngineHandler::scanMessage():FaultBadMessageBuffer2(0 >= inputMessageLength)");
  1100. const char Unknown_SNFMatchFlag = '-';
  1101. const char Panic_SNFMatchFlag = 'p';
  1102. const char Match_SNFMatchFlag = 'm';
  1103. const char White_SNFMatchFlag = 'w';
  1104. const char Final_SNFMatchFlag = 'f';
  1105. void captureMatchRecord(snf_match& M, MatchRecord* R) {
  1106. M.flag = Unknown_SNFMatchFlag;
  1107. M.ruleid = R->RuleId();
  1108. M.symbol = R->RuleGroup();
  1109. M.index = R->MatchStartPosition;
  1110. M.endex = R->MatchEndPosition;
  1111. }
  1112. void snf_SaccadesHandler::applySaccades(EvaluationMatrix* Scanner, vector<unsigned char>& Data) {
  1113. if(NULL == Scanner) return;
  1114. bool isTimeToPeek = (0 >= TimeToPeekCounter);
  1115. if(isTimeToPeek) {
  1116. TimeToPeekCounter = TimeToPeekReset;
  1117. return;
  1118. } else {
  1119. --TimeToPeekCounter;
  1120. }
  1121. vector<saccade> Saccades = grabSaccades();
  1122. for(vector<saccade>::iterator i = Saccades.begin(); i != Saccades.end(); i++) {
  1123. const saccade& s = (*i);
  1124. if(s.start >= Data.size()) break;
  1125. Scanner->evaluateSegment(Data, s.start, s.finish);
  1126. }
  1127. }
  1128. bool isLearnableMatch(MatchRecord* m) {
  1129. bool isGoodSymbol = (0 <= m->RuleGroup() && 64 > m->RuleGroup());
  1130. bool isBeyondAlwaysScan = (snf_SaccadesHandler::AlwaysScanLength < m->MatchEndPosition);
  1131. return (isGoodSymbol && isBeyondAlwaysScan);
  1132. }
  1133. void snf_SaccadesHandler::learnMatches(MatchRecord* Matches) {
  1134. if(NULL == Matches) return;
  1135. vector<saccade> MatchesToLearn;
  1136. saccade WatchForHeaderWhiteRules(0, AlwaysScanLength);
  1137. MatchesToLearn.push_back(WatchForHeaderWhiteRules);
  1138. for(MatchRecord* m = Matches; NULL != m; m = m->NextMatchRecord) {
  1139. if(isLearnableMatch(m)) {
  1140. MatchesToLearn.push_back(
  1141. saccade(
  1142. m->MatchStartPosition,
  1143. m->MatchEndPosition)
  1144. );
  1145. }
  1146. }
  1147. if(0 < MatchesToLearn.size()) {
  1148. lockAndLearn(MatchesToLearn);
  1149. }
  1150. }
  1151. static snf_SaccadesHandler SaccadeBrain;
  1152. static snf_IPStrangerList StrangersList;
  1153. int snf_EngineHandler::scanMessage( // Scan this message (in buffer).
  1154. const unsigned char* inputMessageBuffer, // -- this is the message buffer.
  1155. const int inputMessageLength, // -- this is the length of the buffer.
  1156. const string MessageName, // -- this is the message identifier.
  1157. const int MessageSetupTime, // -- setup time used (for logging).
  1158. const IP4Address MessageSource // -- message source IP (for injection).
  1159. ) {
  1160. ScopeTimer ScanTimeCapture(MyScanData.ScanTime); // Start the scan time clock.
  1161. unsigned char* MessageBuffer = NULL; // Explicitly initialize these two
  1162. int MessageLength = 0; // so the compiler will be happy.
  1163. FaultBadMessageBuffer1(NULL == inputMessageBuffer); // Fault on null message buffer.
  1164. FaultBadMessageBuffer2(0 >= inputMessageLength); // Fault on bad message bfr length.
  1165. // Protect this engine - only one scan at a time per EngineHandler ;-)
  1166. ScopeMutex ScannerIsBusy(MyMutex); // Serialize this...
  1167. // Preliminary job setup.
  1168. // In our pre-processing we may adjust our input buffer so we capture the
  1169. // originals and then use the captured values. For example if we are scanning
  1170. // Communigate message files we will want to skip the communigate headers.
  1171. MessageBuffer = const_cast<unsigned char*>(inputMessageBuffer); // Capture the input buffer.
  1172. MessageLength = inputMessageLength; // Capture the input length.
  1173. MyScanData.clear(); // Clear the scan data.
  1174. MyScanData.ScanSize = MessageLength; // Grab the message length.
  1175. MyScanData.SetupTime = MessageSetupTime; // Capture the setup time.
  1176. if(0 == MyScanData.StartOfJobUTC) { // If the job timestamp is not
  1177. MyScanData.StartOfJobUTC = MyRulebase->MyLOGmgr.Timestamp(); // yet set then set it.
  1178. }
  1179. MyScanData.CallerForcedSourceIP(MessageSource); // Capture the MessageSource if any.
  1180. // Special note about exceptions here...
  1181. // Setting up the filter chain can throw an exception. It can't go in it's own try block or it will
  1182. // be out of scope for the remainder of the function... SO, I've wrapped everything inside of the
  1183. // Lock() in a try block ... and there's a nested one also for scanning the content. The result is
  1184. // that I can put all of the unlock work in the "outer" try block and re-throw anything that's
  1185. // needed.
  1186. snfCFGPacket MyCFGPacket(MyRulebase); // We need this to stay in scope.
  1187. // Set up the filter chain, configure the scanner, and scan the message.
  1188. try {
  1189. if(MyCFGPacket.bad()) { // If it's not there it's a big problem.
  1190. throw Panic("snf_EngineHandler::scanMessage() MyCFGPacket.bad()");
  1191. }
  1192. // Adapt to CGP message files - skip the CGP headers
  1193. MyScanData.MessageFileTypeCGPOn = // Find out if we are expecting
  1194. MyCFGPacket.Config()->MessageFileTypeCGP_on_off; // Communigate message files.
  1195. if(MyScanData.MessageFileTypeCGPOn) { // If we are scanning CGP files:
  1196. while(4 < MessageLength) { // Skip over the CGP headers.
  1197. if( // On Winx systems look for the first
  1198. '\r' == MessageBuffer[0] && // blank line encoded as CRLF CRLF.
  1199. '\n' == MessageBuffer[1] &&
  1200. '\r' == MessageBuffer[2] &&
  1201. '\n' == MessageBuffer[3]
  1202. ) { // If we find it then skip past
  1203. MessageBuffer += 4; // the new line and break out
  1204. MessageLength -= 4; // of the loop.
  1205. break;
  1206. } else // On *nix systems look for the first
  1207. if( // blank line encoded as LF LF.
  1208. '\n' == MessageBuffer[0] &&
  1209. '\n' == MessageBuffer[1]
  1210. ) { // If we find it then skip past
  1211. MessageBuffer += 2; // the blank line and break out
  1212. MessageLength -= 2; // of the loop.
  1213. break;
  1214. }
  1215. else { // If we don't find it then
  1216. ++MessageBuffer; // eat one byte from the buffer
  1217. --MessageLength; // and keep going.
  1218. }
  1219. }
  1220. // At this point our MessagBuffer contains just the message we
  1221. // want to scan.
  1222. MyScanData.ScanSize = MessageLength; // Reset the scan size.
  1223. }
  1224. // Identify this message.
  1225. if( // How do we identify this scan?
  1226. 0 == MessageName.length() || // If no name was provided or
  1227. true == MyCFGPacket.Config()->Scan_Identifier_Force_Message_Id // we are forcing RFC822 IDs then
  1228. ) { // extract the Message-ID from the
  1229. MyScanData.ScanName = extractMessageID(MessageBuffer, MessageLength); // message and use that.
  1230. } else { // If a name was provided and we
  1231. MyScanData.ScanName = MessageName; // are not forcing RFC822 IDs then
  1232. } // use the name provided to us.
  1233. // Set up our filter chain.
  1234. stringstream PrependedHeaders; // Use this to prepend X-Headers.
  1235. FilterChainCBFR IU(MessageBuffer, MessageLength, PrependedHeaders); // Set up the filter chain.
  1236. FilterChainHeaderAnalysis IV(&IU, MyIPTestEngine); // Include header analysis.
  1237. FilterChainBase64 IW(&IV); // Include Base64 decoding.
  1238. FilterChainQuotedPrintable IX(&IW); // Include Quoted Printable decoding.
  1239. FilterChainUrlDecode IY(&IX); // Include URL decoder.
  1240. FilterChainDefunker IZ(&IY); // Include Defunking.
  1241. // Now we set up our scanner and grab the current token matrix.
  1242. if(NULL!=CurrentMatrix) { delete CurrentMatrix; CurrentMatrix=NULL; } // If we have old results, delete them.
  1243. try {
  1244. CurrentMatrix = new EvaluationMatrix(MyCFGPacket.Tokens()); // Allocate a new matrix for this scan.
  1245. } catch(...) { // Check that the allocation worked.
  1246. throw AllocationError("new EvaluationMatrix() ???");
  1247. }
  1248. // Here we get down to it and start scanning the message.
  1249. const char* DebugInfo = "scanMessage() Begin Message Scan"; // If we panic, here we are.
  1250. try {
  1251. // The IPTestEngine has the ability to truncate the message in the filter
  1252. // chain under certain conditions. In order to configure those conditions
  1253. // the IPTestEngine needs to have the configuration data being used for
  1254. // the current scan.
  1255. DebugInfo = "scanMessage() setCFGData()"; // If we panic, here we are.
  1256. MyIPTestEngine.setCFGData(*(MyCFGPacket.Config())); // Setup the CFG data to use.
  1257. // Check processed headers for header directive rules. One of these might
  1258. // include a directive to get the message source IP from a header. If so
  1259. // then MyScanData will have been modified. Also if there are drill-down
  1260. // directives then MyScanData will have been modified to mark any headers
  1261. // that should be ignored -- in this case the IP test used in the filter
  1262. // chain will take appropriate action as it comes across the Received
  1263. // headers that have been marked.
  1264. DebugInfo = "scanMessage() Get Header Directives";
  1265. MyScanData.HeaderDirectiveFlags = 0x00000000; // Clear the header directive flags.
  1266. if(0 < MyCFGPacket.Config()-> // Check to see if we have any
  1267. HeaderDirectivesHandler.HeaderDirectives.size()) { // header directive rules and if we do:
  1268. HeaderFinder HeaderDirectivesParser( // Parse the headers in the message
  1269. &MyScanData, // and update the ScanData using the
  1270. MyCFGPacket.Config()->HeaderDirectivesHandler.HeaderDirectives, // directives in our configuration packet.
  1271. MessageBuffer, // Pass the message as a pointer with
  1272. MessageLength // a specific buffer length.
  1273. );
  1274. MyScanData.HeaderDirectiveFlags = HeaderDirectivesParser(); // Capture the parsed results.
  1275. }
  1276. // Message header rules in earlier versions occasionally failed because there was not
  1277. // a new-line character in front of the very first header. So, now we insert one :-)
  1278. // This allows all header rules to start off with a ^ indicating the start of the line.
  1279. // 20070719_M Added \n to X-snfScanSize: synthetic header.
  1280. // 20070120_M There are some messages where the size is a specific part of
  1281. // the pattern so we will now be emitting this data into the engine. A later
  1282. // version of the engine should handle this kind of thing using a special
  1283. // filter chain module.
  1284. DebugInfo = "scanMessage() ^X-snfScanSize"; // If we panic here we are.
  1285. // Build the scan size info
  1286. PrependedHeaders << "X-snfScanSize: " << MyScanData.ScanSize << "\n"; // and format as an X- header.
  1287. // Add a phantom received header to the top IF the message source has been
  1288. // forced by the caller or by a header directive. After that the normal
  1289. // scanning and header analysis process should pick up the IP as the
  1290. // source of the message. (It will not if the IP is ignored in the GBUdb!)
  1291. DebugInfo = "scanMessage() PhantomReceived"; // If we panic we are here.
  1292. if(0UL != MyScanData.CallerForcedSourceIP()) { // If the caller forced the source IP:
  1293. PrependedHeaders // Make a phantom Received header
  1294. << "Received: Caller.Forced.Source.IP [" // showing that the caller forced
  1295. << (string) MyScanData.CallerForcedSourceIP() << "]\n"; // the source IP.
  1296. } else
  1297. // If not forced by the caller but a
  1298. if(0UL != MyScanData.HeaderDirectiveSourceIP()) { // header directive forced the source IP:
  1299. PrependedHeaders // Make a phantom Received header
  1300. << "Received: Header.Directive.Source.IP [" // showing that a header directive
  1301. << (string) MyScanData.HeaderDirectiveSourceIP() << "]\n"; // established the source IP.
  1302. }
  1303. // Most of the time we will extract the source IP the normal way.
  1304. // If there are other prepended headers to add they should go here.
  1305. /** Add other prepended headers **/
  1306. // 20070719_M Reworked the engine to handle the filter-chain section in
  1307. // a tight loop separately from the scanning section. This should allow
  1308. // for tighter optimization in some cases (less cache thrashing) and also
  1309. // provides for later development of parallel analysis of the pre-filtered
  1310. // data, as well as the ability to output the pre-filtered data for use in
  1311. // rule development and debugging.
  1312. DebugInfo = "scanMessage() IZ.GetByte() ==> FilteredData"; // If we panic we are here.
  1313. unsigned char xb=0;
  1314. MyScanData.FilteredData.clear(); // Clear the FilteredData buffer.
  1315. try { // Watch for exceptions and scan
  1316. for(int a = 0; a < snf_ScanHorizon; a++) // the message through the filter
  1317. MyScanData.FilteredData.push_back(xb=IZ.GetByte()); // chain into the FilteredData buffer.
  1318. } // When we run out of data we will
  1319. catch(const FilterChain::Empty&) {} // get the Empty exception and stop.
  1320. // Scan each byte in the file up to the horizon or the end of the message.
  1321. // If something goes wrong, an exception will be thrown.
  1322. DebugInfo = "scanMessage() EvaluateThis(FilteredData)"; // If we panic, here we are.
  1323. if(false == MyScanData.GBUdbTruncateExecuted) { // If we haven't already truncated:
  1324. //for(int a = 0, b = MyScanData.FilteredData.size(); a < b; a++) // Scan through the filtered data one
  1325. // CurrentMatrix->EvaluateThis(MyScanData.FilteredData[a]); // byte at a time.
  1326. /** 20200618_M Experiment stripping out saccades to check performance hit
  1327. unsigned int fullLength = MyScanData.FilteredData.size();
  1328. SaccadeBrain.applySaccades(CurrentMatrix, MyScanData.FilteredData);
  1329. bool messageNotRecognized = (NULL == CurrentMatrix->ResultList);
  1330. if(messageNotRecognized) {
  1331. CurrentMatrix->evaluateSegment(MyScanData.FilteredData, 0, fullLength);
  1332. SaccadeBrain.learnMatches(CurrentMatrix->ResultList);
  1333. }
  1334. **/
  1335. size_t fullLength = MyScanData.FilteredData.size();
  1336. CurrentMatrix->evaluateSegment(MyScanData.FilteredData, 0, fullLength); // Only do a full scan -- see comment above
  1337. }
  1338. DebugInfo = "scanMessage() Scan Data Complete"; // If we panic, here we are.
  1339. }
  1340. catch(const EvaluationMatrix::BadAllocation&) { // Check for bad allocation during scan.
  1341. throw AllocationError("EvaluationMatrix::BadAllocation");
  1342. }
  1343. catch(const EvaluationMatrix::MaxEvalsExceeded&) { // Check for too many evaluators.
  1344. throw MaxEvals("EvaluationMatrix::MaxEvalsExceeded");
  1345. }
  1346. catch(const EvaluationMatrix::OutOfRange&) { // Check for out of range of (bad) matrix.
  1347. throw BadMatrix("EvaluationMatrix::OutOfRange");
  1348. }
  1349. catch(...){ // In order to prevent thread craziness
  1350. throw Panic(DebugInfo); // throw a Panic.
  1351. } // The mutex will unlock in the outer try.
  1352. }
  1353. // Here is the end of the outer try block. We can catch and rethrow whatever happend
  1354. // and we can also keep our mutex properly managed.
  1355. catch(AllocationError& e) { // Allocation Errors pass through.
  1356. MyRulebase->MyLOGmgr.logThisError( // Log the error.
  1357. MyScanData, "scanMessage()",
  1358. snf_ERROR_ALLOCATION, "ERROR_ALLOCATION"
  1359. );
  1360. throw;
  1361. }
  1362. catch(MaxEvals& e) { // MaxEvals == Panic, with a log.
  1363. MyRulebase->MyLOGmgr.logThisError( // Log the error.
  1364. MyScanData, "scanMessage()",
  1365. snf_ERROR_MAX_EVALS, "ERROR_MAX_EVALS"
  1366. );
  1367. throw;
  1368. }
  1369. catch(BadMatrix& e) { // BadMatrix == Panic, with a log.
  1370. MyRulebase->MyLOGmgr.logThisError( // Log the error.
  1371. MyScanData, "scanMessage()",
  1372. snf_ERROR_BAD_MATRIX, "ERROR_BAD_MATRIX"
  1373. );
  1374. throw;
  1375. }
  1376. catch(Panic& e) { // Panic is panic.
  1377. MyRulebase->MyLOGmgr.logThisError( // Log the error.
  1378. MyScanData, "scanMessage()",
  1379. snf_ERROR_BAD_MATRIX, "ERROR_PANIC"
  1380. );
  1381. throw;
  1382. }
  1383. catch(exception& e) { // Other exceptions.
  1384. MyRulebase->MyLOGmgr.logThisError( // Log the error.
  1385. MyScanData, "scanMessage()",
  1386. snf_ERROR_UNKNOWN, "ERROR_EXCEPTION"
  1387. );
  1388. throw;
  1389. }
  1390. catch(...) { // Anything else == Panic.
  1391. MyRulebase->MyLOGmgr.logThisError( // Log the error.
  1392. MyScanData, "scanMessage()",
  1393. snf_ERROR_UNKNOWN, "ERROR_UNKNOWN"
  1394. );
  1395. throw Panic("snf_EngineHandler::scanMessage() ERROR_UNKNOWN!");
  1396. }
  1397. // At this point, we've completed our scan and we're ready to evaluate our results to find the correct symbol to return.
  1398. ResultsCount = 0; // Reset the count,
  1399. ResultsRemaining = 0; // Remaining count,
  1400. FinalResult = NULL; // Final Result marker,
  1401. ResultCursor = CurrentMatrix -> ResultList; // And cursor position for our results.
  1402. // Now that our result processing gadgets are reset, let's process the results list.
  1403. int const CLEAN_RESULT = 0; // CLEAN means no matches or white.
  1404. int const NO_SYMBOL = 999; // NO_SYMBOL is higher than any SYMBOL
  1405. int S = NO_SYMBOL; // so we start there and work down.
  1406. snf_match TmpSNFMatch; // We'll need a buffer for our matches.
  1407. while(NULL!=ResultCursor) { // While we have records to process...
  1408. captureMatchRecord(TmpSNFMatch, ResultCursor); // grab the next record and evaluate it.
  1409. // Mitigate short-match rulebase events to prevent false positives.
  1410. const size_t minimumPatternLength = 5; // Establish a minimum match length.
  1411. size_t matchSpan = (TmpSNFMatch.endex - TmpSNFMatch.index); // Determine the length of this match.
  1412. bool isShortMatchEvent = (minimumPatternLength > matchSpan); // Identify short-match events.
  1413. bool isPanickedRule = ( // In addition to rule IDs that are
  1414. MyCFGPacket.isRulePanic(TmpSNFMatch.ruleid) || // in the rule-panic list also treat
  1415. isShortMatchEvent // short match events as panic rules.
  1416. );
  1417. bool isVotingCandidate = (false == isPanickedRule); // Panic rules can't vote.
  1418. bool isWhiteRule = (
  1419. MyCFGPacket.Config()->TrainingWhiteRuleHandler.isListed(TmpSNFMatch.ruleid) ||
  1420. 0 == TmpSNFMatch.symbol
  1421. );
  1422. bool isBestResultCode = (TmpSNFMatch.symbol < S);
  1423. // Set an appropriate flag.
  1424. if(isPanickedRule) TmpSNFMatch.flag = Panic_SNFMatchFlag;
  1425. else if(isWhiteRule) TmpSNFMatch.flag = White_SNFMatchFlag;
  1426. else TmpSNFMatch.flag = Match_SNFMatchFlag;
  1427. // Vote for best rule match.
  1428. if(isVotingCandidate && isBestResultCode) {
  1429. FinalResult = ResultCursor;
  1430. S = TmpSNFMatch.symbol;
  1431. }
  1432. // Record this MatchRecord and mMove on to next result.
  1433. MyScanData.MatchRecords.push_back(TmpSNFMatch);
  1434. ResultsCount++;
  1435. ResultCursor=ResultCursor->NextMatchRecord;
  1436. }
  1437. if(NO_SYMBOL != S) { // If a pattern match was detected then
  1438. MyScanData.PatternWasFound = true; // trip the flag and record the
  1439. MyScanData.PatternID = FinalResult->RuleId(); // Rule ID and the
  1440. MyScanData.PatternSymbol = FinalResult->RuleGroup(); // Symbol.
  1441. }
  1442. //// GBUdb Integration ///////////////////////////////////////////////////////
  1443. // To integrate GBUdb we need to generalize the result from the pattern scan.
  1444. PatternResultTypes ScanResultType = NoPattern; // What kind of result have we here?
  1445. if(0 < (MyScanData.HeaderDirectiveFlags & HeaderDirectiveWhite)) { // If a white header directive matched
  1446. ScanResultType = WhitePattern; // then we have a "WhitePattern'.
  1447. } else
  1448. if(MyCFGPacket.Config()->TrainingWhiteRuleHandler.isListed(S)) { // If the pattern was mapped to a white
  1449. ScanResultType = WhitePattern; // rule group then we have a 'WhitePattern'.
  1450. } else
  1451. if(CLEAN_RESULT == S) { // If there was a standard white rule
  1452. ScanResultType = WhitePattern; // result then we have a 'WhitePattern'.
  1453. } else
  1454. if(NO_SYMBOL == S) { // If there was no pattern match then
  1455. ScanResultType = NoPattern; // we have 'NoPattern'.
  1456. } else
  1457. if(63 == S) { // If the pattern was a standard IP rule
  1458. ScanResultType = IPPattern; // then we have an 'IPPattern'.
  1459. } else
  1460. if(62 >= S) { // In general, other nonzer rule groups
  1461. ScanResultType = BlackPattern; // indicate we have a 'BlackPatter'.
  1462. } else
  1463. if(63 < S) { // Any pattern number > 63 is special.
  1464. ScanResultType = AboveBandPattern; // Any of these are an 'AboveBandPattern'
  1465. }
  1466. if(MyScanData.FoundSourceIP()) { // We need an identified IP source.
  1467. // Train the GBUdb based on our pattern matching results.
  1468. // Evaluate our training conditions.
  1469. bool TrainingIsTurnedOn = MyCFGPacket.Config()->GBUdbTrainingOn_Off;
  1470. bool MessageWasNotTruncated = (false == MyScanData.GBUdbTruncateExecuted);
  1471. bool ThereIsNoBypassHeaderDirective = (0 == (MyScanData.HeaderDirectiveFlags & HeaderDirectiveBypass));
  1472. bool ThereIsNoBypassResultCodeRule = (false == MyCFGPacket.Config()->TrainingBypassRuleHandler.isListed(S));
  1473. bool ThereIsNoImpliedBypassDirective = (Ignore != (MyScanData.SourceIPRecord().GBUdbData.Flag()));
  1474. // If these conditions are favorable then train the GBUdb.
  1475. if( // Check to see if training is enabled.
  1476. TrainingIsTurnedOn && // If it is turned on AND
  1477. MessageWasNotTruncated && // The message was not truncated AND
  1478. ThereIsNoBypassHeaderDirective && // There is NO Bypass header directive AND
  1479. ThereIsNoBypassResultCodeRule && // There is NO Bypass result code rule AND
  1480. ThereIsNoImpliedBypassDirective // There is NO Implied bypass directive:
  1481. ) {
  1482. // GBUdb training is enabled.
  1483. bool discoveredNewIP = false;
  1484. IP4Address theSourceIP = MyScanData.SourceIPRecord().IP;
  1485. switch(ScanResultType) { // Evaluate the scan result.
  1486. case NoPattern: // On no pattern (benefit of doubt) or
  1487. case WhitePattern: { // a white pattern:
  1488. GBUdbRecord thisRecord = // Grab the GBUdb record for later
  1489. MyRulebase->MyGBUdb.addGood( // then add a good count to the
  1490. theSourceIP); // source IP.
  1491. discoveredNewIP = (0 == thisRecord.Bad() && 1 == thisRecord.Good());
  1492. if(discoveredNewIP) { // New IPs are strangers.
  1493. StrangersList.addStranger(theSourceIP); // Add them to the list
  1494. thisRecord.Bad(thisRecord.Good()); // and set their reputation
  1495. MyRulebase->MyGBUdb.setRecord(theSourceIP, thisRecord); // to 50/50 at best.
  1496. } else
  1497. if( // Known IPs that are getting
  1498. thisRecord.Good() > thisRecord.Bad() && // an advantage but are on the
  1499. StrangersList.isStranger(theSourceIP) // strangers list get put back
  1500. ) { // to a 50/50 reputation.
  1501. unsigned int equalizationValue = thisRecord.Good();
  1502. if(1 < equalizationValue) equalizationValue = equalizationValue / 2;
  1503. thisRecord.Bad(equalizationValue);
  1504. thisRecord.Good(equalizationValue);
  1505. MyRulebase->MyGBUdb.setRecord(theSourceIP, thisRecord);
  1506. }
  1507. break;
  1508. }
  1509. case BlackPattern: { // On a black pattern:
  1510. GBUdbRecord thisRecord = // Grab the GBUdb record for later
  1511. MyRulebase->MyGBUdb.addBad( // Add a bad count to the source IP
  1512. MyScanData.SourceIPRecord().IP); // in the GBUdb.
  1513. discoveredNewIP = (1 == thisRecord.Bad() && 0 == thisRecord.Good());
  1514. if(discoveredNewIP) StrangersList.addStranger(theSourceIP);
  1515. break;
  1516. }
  1517. default: break; // In all other cases, don't train.
  1518. }
  1519. }
  1520. // GBUdb Training Is Complete
  1521. // At this point our SourceIPRange tells us exactly how to evaluate
  1522. // the source IP for this message.
  1523. switch(MyScanData.SourceIPRange()) {
  1524. case White: { // If the IP was in the white zone
  1525. MyScanData.GBUdbWhiteTriggered = true; // mark that down.
  1526. if(MyCFGPacket.Config()->WhiteRangeHandler.On_Off) { // If we're also turned on then
  1527. if( // do we need to force the symbol?
  1528. BlackPattern == ScanResultType || // We do if the pattern scan resulted
  1529. IPPattern == ScanResultType // in a black or IPblack match.
  1530. ) { // If we must force a white result:
  1531. S = MyCFGPacket.Config()->WhiteRangeHandler.Symbol; // force the symbol and
  1532. MyScanData.GBUdbWhiteSymbolForced = true; // record that it was done.
  1533. }
  1534. // AutoPanic
  1535. int AutoPanicRangeLowerBound = // Calculate the current lower bound
  1536. MyRulebase->MyLOGmgr.LatestRuleID() - // for rule id's that are eligible to
  1537. MyCFGPacket.Config()->gbudb_regions_white_panic_rule_range; // trigger auto-panics.
  1538. if(BlackPattern == ScanResultType || IPPattern == ScanResultType) { // Was there a pattern/source conflict?
  1539. MyScanData.GBUdbPatternSourceConflict = true; // Record the event.
  1540. if(MyScanData.PatternID > AutoPanicRangeLowerBound) { // If the pattern ID is in range then
  1541. MyScanData.GBUdbAutoPanicTriggered = true; // record that the AutoPanic triggered.
  1542. if(MyCFGPacket.Config()->gbudb_regions_white_panic_on_off) { // If rule panics are turned on then
  1543. MyScanData.GBUdbAutoPanicExecuted = true; // indicate we are executing an autopanic.
  1544. MyRulebase->addRulePanic(MyScanData.PatternID); // Add the rule panic.
  1545. }
  1546. }
  1547. }
  1548. }
  1549. break;
  1550. }
  1551. case Normal: { // If the IP is normal...
  1552. MyScanData.GBUdbNormalTriggered = true; // Count the event.
  1553. break; // That's all.
  1554. }
  1555. case New: {
  1556. break;
  1557. }
  1558. case Caution: { // If the IP is in the caution range.
  1559. MyScanData.GBUdbCautionTriggered = true; // Track that this range fired.
  1560. if(
  1561. MyCFGPacket.Config()->CautionRangeHandler.On_Off && // If we're also turned on and there
  1562. NoPattern == ScanResultType // is no pattern match then
  1563. ) { // we will override the scan result:
  1564. S = MyCFGPacket.Config()->CautionRangeHandler.Symbol; // set the symbol as configured and
  1565. MyScanData.GBUdbCautionSymbolForced = true; // record that it was done.
  1566. }
  1567. break;
  1568. }
  1569. // Truncate is a kind of uber-black, so we do some weirdness here.
  1570. // If Truncate happens, then black was triggered by definition. In
  1571. // peek cases or if Truncate is turned off then Truncate might not
  1572. // execute-- when that happens we need to fall back to Black behavior.
  1573. case Truncate: // If the IP was in the truncate range
  1574. case Black: { // and/or If the IP is in the black range
  1575. MyScanData.GBUdbBlackTriggered = true; // mark that down.
  1576. if(MyScanData.GBUdbTruncateExecuted) { // If the truncate action was executed
  1577. S = MyCFGPacket.Config()->gbudb_regions_black_truncate_symbol; // we set the output symbol accordingly.
  1578. } else // Truncate overrides black.. but if
  1579. if( // Black is in charge do this...
  1580. MyCFGPacket.Config()->BlackRangeHandler.On_Off && // If black action is turned on and there
  1581. NoPattern == ScanResultType // is no pattern match then
  1582. ) { // we will override the scan data:
  1583. S = MyCFGPacket.Config()->BlackRangeHandler.Symbol; // set the symbol as configured and
  1584. MyScanData.GBUdbBlackSymbolForced = true; // record that it was done.
  1585. }
  1586. // Now that all of the overrides have been handled we can handle
  1587. // sampling. When a black IP is detected and a pattern match is not
  1588. // then we may sample the data.
  1589. int BlackSampleRate = // Grab the sample rate to make the
  1590. MyCFGPacket.Config()->gbudb_regions_black_sample_grab_one_in; // logic clearer.
  1591. bool SampleThresholdReached = // Check the spam probability of the
  1592. (MyCFGPacket.Config()->gbudb_regions_black_sample_probability <= // source IP against the configuration
  1593. MyScanData.SourceIPRecord().GBUdbData.Probability()); // to see if this IP is a candidate.
  1594. if( // Should we sample?
  1595. false == MyScanData.GBUdbTruncateExecuted && // If this was not a truncation and
  1596. NoPattern == ScanResultType && // No pattern match was found and
  1597. SampleThresholdReached && // We reached out sample threshold and
  1598. MyRulebase->MyLOGmgr.OkToSample(BlackSampleRate) // It's ok for us to sample this round
  1599. ) { // then our sampling mechanism is triggerd.
  1600. MyScanData.GBUdbSampleTriggered = true; // Mark down that event.
  1601. if(MyCFGPacket.Config()->gbudb_regions_black_sample_on_off) { // If sampling is turned on then
  1602. MyScanData.GBUdbSampleExecuted = true; // we will be sampling this data.
  1603. if(MyCFGPacket.Config()->gbudb_regions_black_sample_passthrough) { // If sampling by passthrough then
  1604. S = MyCFGPacket.Config()-> // Force the symbol value to passthrough
  1605. gbudb_regions_black_sample_passthrough_symbol; // (usually 0 - same as CLEAN).
  1606. } else { // If sampling internally then
  1607. MyRulebase->MyNETmgr.sendSample( // send this message as a sample.
  1608. (*(MyCFGPacket.Config())), // Pass our current config info,
  1609. MyScanData, // our scan data,
  1610. MessageBuffer, // and the message itself.
  1611. MessageLength
  1612. );
  1613. }
  1614. }
  1615. }
  1616. break;
  1617. }
  1618. case Unknown: // Unknown - most likely we couldn't
  1619. default: { // find a usable source.
  1620. break; // Do nothing.
  1621. }
  1622. }
  1623. } // End of IP source depended work (GBUdbOverrides)
  1624. // At this point we know the final result of our scan
  1625. // and the number of results we have. It's time to set up our result
  1626. // processing widgets for further query and return the result of this scan.
  1627. ResultCursor = CurrentMatrix -> ResultList; // Starting at the top of the list
  1628. ResultsRemaining = ResultsCount; // with all of the results ahead of us.
  1629. if(NO_SYMBOL==S) S = CLEAN_RESULT; // When there were no results, CLEAN
  1630. MyScanData.CompositeFinalResult = S; // Record what we will return.
  1631. if( // Prepare our final result.
  1632. CLEAN_RESULT == S && // If we have a clean result code
  1633. ScanResultType != WhitePattern && // and it wasn't forced by a white
  1634. false == MyScanData.GBUdbWhiteSymbolForced) { // rule or white GBUdb then we mark
  1635. TmpSNFMatch.flag = 'c'; // the final record Clean.
  1636. } else { // Otherwise we mark the final record
  1637. TmpSNFMatch.flag = 'f'; // as Final - meaning deliberately zero.
  1638. }
  1639. TmpSNFMatch.index = 0; // Our index is charater zero.
  1640. TmpSNFMatch.endex = CurrentMatrix->CountOfCharacters - 1; // Our endex is the end of the message.
  1641. TmpSNFMatch.symbol = MyScanData.CompositeFinalResult; // Our symbol is in CompositeFinal.
  1642. // The rule id is dependent on what's happened...
  1643. if( // If the symbol has been forced...
  1644. MyScanData.GBUdbTruncateExecuted || // Was it a Truncate-IP scan?
  1645. MyScanData.GBUdbWhiteSymbolForced || // Was it a White-IP scan?
  1646. MyScanData.GBUdbBlackSymbolForced || // Was it a Black-IP scan?
  1647. MyScanData.GBUdbCautionSymbolForced || // Was it a Caution-IP scan?
  1648. NULL == FinalResult // OR there was no valid match
  1649. ) { // then our rule id will be
  1650. TmpSNFMatch.ruleid = 0; // ZERO.
  1651. } else { // Normally the rule id will be
  1652. TmpSNFMatch.ruleid = FinalResult->RuleId(); // that of the winning pattern match.
  1653. }
  1654. MyScanData.MatchRecords.push_back(TmpSNFMatch); // Push our final entry onto the list.
  1655. MyScanData.MatchRecordsCursor = MyScanData.MatchRecords.begin(); // Reset the delivery system to the
  1656. MyScanData.MatchRecordsDelivered = 0; // beginning of the results list.
  1657. MyScanData.ScanDepth = CurrentMatrix->MaximumCountOfEvaluators; // Capture the scan depth.
  1658. MyScanData.ScanTime.stop(); // Stop the scan time clock.
  1659. MyRulebase->MyLOGmgr.logThisScan((*(MyCFGPacket.Config())), MyScanData); // Log the data from this scan.
  1660. // Since V2-9rc19 of this engine, the Engine mutex and snfCFGPacket handle
  1661. // their own cleanup when this call goes out of scope. ScannerIsBusy(MyMutex)
  1662. // will unlock() on destruction and snfCFGPacket will MyRulebase->drop().
  1663. return S; // Return the final scan result.
  1664. }
  1665. int snf_EngineHandler::getResults(snf_match* MatchBuffer){ // Get the next match buffer.
  1666. ScopeMutex SerializeThis(MyMutex); // Serialize this...
  1667. if(NULL == MatchBuffer) { // If we were given the reset signal
  1668. MyScanData.MatchRecordsCursor = MyScanData.MatchRecords.begin(); // Move the cursor to the beginning
  1669. MyScanData.MatchRecordsDelivered = 0; // and reset the delivered count.
  1670. } else { // If we are in delivery mode and
  1671. if(MyScanData.MatchRecords.end() != MyScanData.MatchRecordsCursor) { // there are more to deliver then
  1672. (*MatchBuffer) = (*MyScanData.MatchRecordsCursor); // deliver the current match and
  1673. ++MyScanData.MatchRecordsCursor; // move on to the next. Be sure to
  1674. ++MyScanData.MatchRecordsDelivered; // count this one as delivered.
  1675. }
  1676. }
  1677. return MyScanData.MatchRecords.size() - MyScanData.MatchRecordsDelivered; // Return a count of unseen records.
  1678. }
  1679. int snf_EngineHandler::getDepth(){ // Get the scan depth.
  1680. ScopeMutex SerializeThis(MyMutex); // Protect our reading.
  1681. return MyScanData.ScanDepth; // Return the latest scan depth.
  1682. }
  1683. const string snf_EngineHandler::getClassicLog() { // Get classic log entries for last scan.
  1684. ScopeMutex SerializeThis(MyMutex); // Serialize this...
  1685. return MyScanData.ClassicLogText; // Return the log text.
  1686. }
  1687. const string snf_EngineHandler::getXMLLog() { // Get XML log entries or last scan.
  1688. ScopeMutex SerializeThis(MyMutex); // Serialize this...
  1689. return MyScanData.XMLLogText; // Return the log text.
  1690. }
  1691. const string snf_EngineHandler::getXHDRs() { // Get XHDRs for last scan.
  1692. ScopeMutex SerializeThis(MyMutex); // Serialize this...
  1693. return MyScanData.XHDRsText; // Return the XHeaders text.
  1694. }
  1695. //// Multi Engine Handler Methods
  1696. // snf_RoundRulebaseCursor()
  1697. // Returns the next rulebase slot id wrapping around to zero.
  1698. int snf_MultiEngineHandler::RoundRulebaseCursor(){ // Return the next Rulebase handle
  1699. RulebaseCursor++; // Increase the cursor.
  1700. if(snf_MAX_RULEBASES<=RulebaseCursor) // If we've reached the end of the array
  1701. RulebaseCursor=0; // then we start back at zero.
  1702. return RulebaseCursor; // Return the new handle candidate.
  1703. }
  1704. // snf_RoundEngineCursor()
  1705. // Returns the next engine slot id wrapping around to zero.
  1706. int snf_MultiEngineHandler::RoundEngineCursor(){ // Return the next Engine handle candidate.
  1707. EngineCursor++; // Increase the cursor.
  1708. if(snf_MAX_SCANNERS<=EngineCursor) // If we've reached the end of the array
  1709. EngineCursor=0; // then we start back at zero.
  1710. return EngineCursor; // Return the new handle candidate.
  1711. }
  1712. snf_MultiEngineHandler::~snf_MultiEngineHandler(){ // Clean up, safety check, shut down.
  1713. RulebaseScan.lock(); // Lock both the rulebase and
  1714. EngineScan.lock(); // engine scan rulebases.
  1715. RulebaseCursor = EngineCursor = SHUTDOWN; // Set the cursors to the FINISHED value.
  1716. // The handlers in the arrays will all get closed by their destructors.
  1717. // The SHUTDOWN value in the cursors will force any errant threads to get no love.
  1718. RulebaseScan.unlock();
  1719. EngineScan.unlock();
  1720. }
  1721. // snf_OpenRulebase()
  1722. // Grab the first available rulebse handler and light it up.
  1723. int snf_MultiEngineHandler::OpenRulebase(const char* path, const char* licenseid, const char* authentication){
  1724. RulebaseScan.lock(); // Serialize this.
  1725. if(SHUTDOWN==RulebaseCursor) { // Not ok to open after shutdown.
  1726. RulebaseScan.unlock();
  1727. throw Panic("snf_MultiEngineHandler::OpenRulebase() No open after shutdown");
  1728. }
  1729. int Handle = RoundRulebaseCursor(); // Grab the next hanlder on the list.
  1730. if(RulebaseHandlers[Handle].isReady()) { // Check to see if it's already in use. If so,
  1731. int wherewasi = Handle; // keep track of where we started.
  1732. while(RulebaseHandlers[(Handle=RoundRulebaseCursor())].isReady()){ // Loop to find an free handler.
  1733. if(wherewasi==Handle) { // If we get back where we started
  1734. RulebaseScan.unlock(); // Unlock the Rulebase Scanning process
  1735. throw TooMany("snf_MultiEngineHandler::OpenRulebase() Too Many Open"); // and tell the caller Too Many are open.
  1736. }
  1737. }
  1738. }
  1739. // Now we have a Handle to a free RulebaseHandler. Time to open it up.
  1740. try {
  1741. RulebaseHandlers[Handle].open(path,licenseid,authentication); // Try to open the handler.
  1742. } // If an exception is thrown...
  1743. catch(snf_RulebaseHandler::AuthenticationError& e) // Catch and re-throw the appropriate
  1744. { RulebaseScan.unlock(); throw AuthenticationError(e.what()); } // exception.
  1745. catch(snf_RulebaseHandler::AllocationError& e)
  1746. { RulebaseScan.unlock(); throw AllocationError(e.what()); }
  1747. catch(snf_RulebaseHandler::FileError& e)
  1748. { RulebaseScan.unlock(); throw FileError(e.what()); }
  1749. catch(snf_RulebaseHandler::Busy& e)
  1750. { RulebaseScan.unlock(); throw Panic(e.what()); } // Wasn't busy above!! Shoudn't be here!!!
  1751. catch(exception& e)
  1752. { RulebaseScan.unlock(); throw; }
  1753. catch(...) {
  1754. RulebaseScan.unlock();
  1755. throw Panic("snf_MultiEngineHandler::OpenRulebase() ???");
  1756. }
  1757. RulebaseScan.unlock(); // If everything went well then UnLock
  1758. return Handle; // and return the happy new handle.
  1759. }
  1760. // snf_RefreshRulebase()
  1761. // Reload the rulebase associated with the handler.
  1762. void snf_MultiEngineHandler::RefreshRulebase(int RulebaseHandle){ // Refreshing a rulebase (Not Serialized)
  1763. try {
  1764. RulebaseHandlers[RulebaseHandle].refresh(); // Try to refresh the rulebase.
  1765. } // Catch and rethrow any exceptions.
  1766. catch(snf_RulebaseHandler::AuthenticationError& e) {
  1767. throw AuthenticationError(e.what());
  1768. }
  1769. catch(snf_RulebaseHandler::AllocationError& e) {
  1770. throw AllocationError(e.what());
  1771. }
  1772. catch(snf_RulebaseHandler::FileError& e) {
  1773. throw FileError(e.what());
  1774. }
  1775. catch(snf_RulebaseHandler::Busy& e) {
  1776. throw Busy(e.what());
  1777. }
  1778. catch(exception& e) {
  1779. throw;
  1780. }
  1781. catch(...) {
  1782. throw Panic("snf_MultiEngineHandler::RefreshRulebase() ???");
  1783. }
  1784. }
  1785. // snf_CloseRulebase()
  1786. // Shut down this Rulebase handler.
  1787. void snf_MultiEngineHandler::CloseRulebase(int RulebaseHandle){ // Closing a rulebase handler
  1788. RulebaseScan.lock(); // Serialize this - the handler changes state.
  1789. try { // Try to close the handler.
  1790. RulebaseHandlers[RulebaseHandle].close();
  1791. }
  1792. catch(snf_RulebaseHandler::Busy& e) { // A busy throw we can understand.
  1793. RulebaseScan.unlock(); throw Busy(e.what());
  1794. }
  1795. catch(exception& e) { // Other exceptions? rethrow.
  1796. RulebaseScan.unlock(); throw;
  1797. }
  1798. catch(...) { // Any other throw is big trouble.
  1799. RulebaseScan.unlock();
  1800. throw Panic("snf_MultiEngineHandler::CloseRulebase() ???");
  1801. }
  1802. RulebaseScan.unlock(); // When done, unlock the Rulebase Scan process.
  1803. }
  1804. // snf_OpenEngine()
  1805. // Grab the first available Engine handler and light it up
  1806. int snf_MultiEngineHandler::OpenEngine(int RulebaseHandle){
  1807. EngineScan.lock(); // Serialize this.
  1808. if(SHUTDOWN==EngineCursor) { // Not ok to open after shutdown.
  1809. EngineScan.unlock();
  1810. throw Panic("snf_MultiEngineHandler::OpenEngine() No open after shutdwon");
  1811. }
  1812. int Handle = RoundEngineCursor(); // Grab the next hanlder on the list.
  1813. if(EngineHandlers[Handle].isReady()) { // Check to see if it's already in use. If so,
  1814. int wherewasi = Handle; // keep track of where we started.
  1815. while(EngineHandlers[(Handle=RoundEngineCursor())].isReady()){ // Loop to find an free handler.
  1816. if(wherewasi==Handle) { // If we get back where we started
  1817. EngineScan.unlock(); // Unlock the Rulebase Scanning process
  1818. throw TooMany("snf_MultiEngineHandler::OpenEngine() too many open"); // and tell the caller Too Many are open.
  1819. }
  1820. }
  1821. }
  1822. // Now we have a Handle to a free RulebaseHandler. Time to open it up.
  1823. try {
  1824. EngineHandlers[Handle].open(&RulebaseHandlers[RulebaseHandle]); // Try to open the handler.
  1825. } // If an exception is thrown...
  1826. catch(snf_EngineHandler::AllocationError& e) // Catch and rethrow as appropriate.
  1827. { EngineScan.unlock(); throw AllocationError(e.what()); }
  1828. catch(snf_EngineHandler::Busy& e)
  1829. { EngineScan.unlock(); throw Panic(e.what()); } // Not busy above should not be busy now!!!
  1830. catch(exception& e) {
  1831. EngineScan.unlock();
  1832. throw;
  1833. }
  1834. catch(...) {
  1835. EngineScan.unlock();
  1836. throw Panic("snf_MultiEngineHandler::OpenEngine() ???");
  1837. }
  1838. EngineScan.unlock(); // If everything went well then UnLock
  1839. return Handle; // and return the happy new handle.
  1840. }
  1841. // snf_CloseEngine()
  1842. // Shut down this Engine handler.
  1843. void snf_MultiEngineHandler::CloseEngine(int EngineHandle){ // Closing an engine handler.
  1844. EngineScan.lock(); // Serialize this, the object changes states.
  1845. try {
  1846. EngineHandlers[EngineHandle].close(); // Try closing the handler.
  1847. }
  1848. catch(snf_EngineHandler::AllocationError& e) // Catch and throw any exceptions as needed.
  1849. { EngineScan.unlock(); throw AllocationError(e.what()); }
  1850. catch(snf_EngineHandler::Busy& e)
  1851. { EngineScan.unlock(); throw Busy(e.what()); }
  1852. catch(exception& e) {
  1853. EngineScan.unlock();
  1854. throw;
  1855. }
  1856. catch(...) {
  1857. EngineScan.unlock();
  1858. throw Panic("snf_MultiEngineHandler::CloseEngine() ???");
  1859. }
  1860. EngineScan.unlock(); // Unlock when we're closed.
  1861. }
  1862. // snf_Scan()
  1863. // Scan the MessageBuffer with this Engine.
  1864. int snf_MultiEngineHandler::Scan(int EngineHandle, const unsigned char* MessageBuffer, int MessageLength){
  1865. // NOT serialized. Many scans at once, presumably one scan engine per thread.
  1866. int ScanResult; // ScanResult stays in scope.
  1867. try {
  1868. ScanResult=EngineHandlers[EngineHandle]
  1869. .scanMessage(MessageBuffer,MessageLength); // Try the scan on the given engine.
  1870. }
  1871. catch(snf_EngineHandler::AllocationError& e) { // Re-throw any exceptions as needed.
  1872. throw AllocationError(e.what());
  1873. }
  1874. catch(snf_EngineHandler::Busy& e) { throw Busy(e.what()); }
  1875. catch(exception& e) { throw; }
  1876. catch(...) { throw Panic("snf_MultiEngineHandler::Scan() ???"); }
  1877. return ScanResult; // Return the results.
  1878. }
  1879. // The Engine prvides detailed match results through this function.
  1880. int snf_MultiEngineHandler::getResults(int EngineHandle, snf_match* matchbfr){
  1881. // NOT serialized. Many scans at once, presumably one scan engine per thread.
  1882. int ResultCount; // ResultCount stays in scope.
  1883. try {
  1884. ResultCount=EngineHandlers[EngineHandle].getResults(matchbfr); // Try the scan on the given engine.
  1885. }
  1886. catch(snf_EngineHandler::AllocationError& e) { // Re-throw any exceptions as needed.
  1887. throw AllocationError(e.what());
  1888. }
  1889. catch(snf_EngineHandler::Busy& e) { throw Busy(e.what()); }
  1890. catch(exception& e) { throw; }
  1891. catch(...) { throw Panic("snf_MultiEngineHandler::getResults() ???"); }
  1892. return ResultCount; // Return the results.
  1893. }
  1894. // The Engine provies the scan depth through this function.
  1895. int snf_MultiEngineHandler::getDepth(int EngineHandle){
  1896. // NOT serialized. Many scans at once, presumably one scan engine per thread.
  1897. int DepthResult; // ScanResult stays in scope.
  1898. try {
  1899. DepthResult=EngineHandlers[EngineHandle].getDepth(); // Try the scan on the given engine.
  1900. }
  1901. catch(snf_EngineHandler::AllocationError& e) { // Re-throw any exceptions as needed.
  1902. throw AllocationError(e.what());
  1903. }
  1904. catch(snf_EngineHandler::Busy& e) { throw Busy(e.what()); }
  1905. catch(exception& e) { throw; }
  1906. catch(...) { throw Panic("snf_MultiEngineHandler::getDepth() ???"); }
  1907. return DepthResult; // Return the results.
  1908. }