Nevar pievienot vairāk kā 25 tēmas Tēmai ir jāsākas ar burtu vai ciparu, tā var saturēt domu zīmes ('-') un var būt līdz 35 simboliem gara.

snf_engine.cpp 44KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791
  1. // snf_engine.cpp
  2. //
  3. // (C) 1985-2004 MicroNeil Research Corporation
  4. // (C) 2005-2009 ARM Research Labs, LLC
  5. // See www.armresearch.com for the copyright terms.
  6. //
  7. // Derived from original work on cellular automation for complex pattern
  8. // reflex engine 1985 Pete McNeil (Madscientist)
  9. //
  10. // Derived from rapid scripting engine (token matrix) implementation 1987
  11. //
  12. // 20040419 _M Adding Verify() method. Beginning with version 2-3 of Message Sniffer
  13. // we are embedding a Mangler digest of the rulebase file. The Verify() method reconstructs
  14. // the digest and compares it. This ensures that no part of the rulebase file can be
  15. // corrupted without the snf2check utility detecting the problem. Prior to this version
  16. // it was possible to have undetected corruption in the middle of the rulebase file. The
  17. // Mangler digest will prevent that.
  18. // 20030130 _M Added testing section in TokenMatrix to throw an exeption if the file
  19. // is too small to be a valid matrix. The value is calculated based on the idea that a
  20. // valid matrix will have been encrypted in two segments so the file must be at least
  21. // as large as these two segments. This is intended to solve the zero-length-rulebase
  22. // bug where an access violation would occur if the file was of zero length.
  23. // 20021030 _M Creation of snf_engine module by dragging the sniffer pattern matching engine out
  24. // of the sniffer.cpp file.
  25. #include <unistd.h>
  26. #include <cstdio>
  27. #include <cctype>
  28. #include <ctime>
  29. #include <cstdlib>
  30. #include <fstream>
  31. #include <iostream>
  32. #include <string>
  33. #include "mangler.hpp"
  34. #include "snf_engine.hpp"
  35. using namespace std;
  36. ///////////////////////////////////////////////////////////////////////////////////////////
  37. // BEGIN IMPLEMENTATIONS //////////////////////////////////////////////////////////////////
  38. ///////////////////////////////////////////////////////////////////////////////////////////
  39. ///////////////////////////////////////////////////////////////////////////////////////////
  40. // Token Matrix Implementations ///////////////////////////////////////////////////////////
  41. // TokenMatrix::Load(filename)
  42. void TokenMatrix::Load(string& FileName) { // Initialize using a string for file name.
  43. Load(FileName.c_str()); // Convert the string to a null terminated
  44. } // char* and call the function below.
  45. void TokenMatrix::Load(const char* FileName) { // Initializes the token matrix by file name.
  46. ifstream MatrixFile(FileName,ios::binary); // Open the file.
  47. if(MatrixFile == NULL || MatrixFile.bad()) // If anything is wrong with the file
  48. throw BadFile("TokenMatrix::Load()(MatrixFile==NULL || MatrixFile.bad())"); // then throw a bad file exception.
  49. Load(MatrixFile); // Load the matrix from the file.
  50. MatrixFile.close(); // Be nice and clean up our file.
  51. }
  52. // TokenMatrix::Load(stream)
  53. void TokenMatrix::Load(ifstream& F) { // Initializes the token matrix from a file.
  54. MatrixSize = 0; // Clear out the old Matrix Size and array.
  55. if(Matrix) delete Matrix; // that is, if there is an array.
  56. F.seekg(0,ios::end); // Find the end of the file.
  57. MatrixSize = F.tellg() / sizeof(Token); // Calculate how many tokens.
  58. F.seekg(0); // Go back to the beginning.
  59. if(MatrixSize < MinimumValidMatrix) // If the matrix file is too small then
  60. throw BadMatrix("TokenMatrix::Load() (MatrixSize < MinimumValidMatrix)"); // we must reject it.
  61. Matrix = new Token[MatrixSize]; // Allocate an array of tokens.
  62. if(Matrix == NULL) // Check for an allocation error.
  63. throw BadAllocation("TokenMatrix::Load() Matrix == NULL)"); // and throw an exception if it happens.
  64. F.read( // Now read the file into the allocated
  65. reinterpret_cast<char*>(Matrix), // matrix by recasting it as a character
  66. (MatrixSize * sizeof(Token))); // buffer of the correct size.
  67. if(F.bad()) // If there were any problems reading the
  68. throw BadMatrix("TokenMatrix::Load() (F.bad())"); // matrix then report the bad matrix.
  69. }
  70. // TokenMatrix::Validate(key)
  71. void TokenMatrix::Validate(string& SecurityKey) { // Decrypts and validates the matrix.
  72. MANGLER ValidationChecker; // Create a mangler engine for validation.
  73. // In order to do the validation we must look at the token matrix as a sequence of bytes.
  74. // We will be decrypting the first and last SecurtySegmentSize of this sequence and then
  75. // detecting wether the appropriate security key has been properly encrypted in the end.
  76. // If we find everything as it should be then we can be sure that the two segments have
  77. // not been tampered with and that we have the correct security key.
  78. unsigned char* TokensAsBytes = reinterpret_cast<unsigned char*>(Matrix);
  79. int BytesInTokenMatrix = (MatrixSize * sizeof(Token));
  80. // Now that we have all of that stuff let's initialize our ValidationChecker.
  81. // Note that the length of our security key is always 24 bytes. The license
  82. // id is 8 bytes, the authentication code is 16 bytes. We don't bother to check
  83. // here because if it's wrong then nothing will decrypt and we'll have essentially
  84. // the same result. Note also that on the end of the rule file we pad this
  85. // encrypted security id with nulls so that we can create a string from it easily
  86. // and so that we have precisely 32 bytes which is the same size as 4 tokens.
  87. //
  88. // Note: The 32 byte value is in SecurityKeyBufferSize. This means that we can
  89. // accept security keys up to 31 bytes in length. We need the ending null to
  90. // assure our null terminated string is as expected. The security key block must
  91. // match up with the edges of tokens in the matrix so we pad the end with nulls
  92. // when encoding the security key in the encoded file.
  93. int SecurityKeyLength = SecurityKey.length(); // For the length of our key
  94. for(int a=0;a<SecurityKeyLength;a++) // feed each byte through the
  95. ValidationChecker.Encrypt(SecurityKey.at(a)); // mangler to evolve the key
  96. // state.
  97. // Now we're ready to decrypt the matrix... We start with the first segment.
  98. for(int a=0;a<SecuritySegmentSize;a++) // For the length of the segment
  99. TokensAsBytes[a] = // replace each byte with the
  100. ValidationChecker.Decrypt(TokensAsBytes[a]); // decrypted byte.
  101. // Next we decrypt the last security segment...
  102. for(int a= BytesInTokenMatrix - SecuritySegmentSize; a<BytesInTokenMatrix; a++)
  103. TokensAsBytes[a] =
  104. ValidationChecker.Decrypt(TokensAsBytes[a]);
  105. // Now that we've done this we should find that our SecurityKey is at the end
  106. // of the loaded token matrix... Let's look and find out shall we?!!!
  107. unsigned char* SecurityCheckKey = // Reference the check
  108. & TokensAsBytes[BytesInTokenMatrix-SecurityKeyBufferSize]; // space in the matrix.
  109. SecurityCheckKey[SecurityKeyBufferSize-1] = 0; // Add a safety null just in case.
  110. string SecurityCheck((char*)SecurityCheckKey); // Make a string.
  111. // By now we should have a SecurityCheck string to compare to our SecurityKey.
  112. // If they match then we know everything worked out and that our token matrix has
  113. // been decrypted properly. This is also a good indication that our token matrix
  114. // is not incomplete since if it were the decryption wouldn't work. Saddly, we
  115. // don't have the computing cycles to decrypt the entire file - so we won't be
  116. // doing that until we can load it in a server/daemon and then reuse it over and
  117. // over... Once that happens we will be able to detect tampering also.
  118. if(SecurityKey != SecurityCheck) // If the security keys don't match
  119. throw BadMatrix("TokenMatrix::Validate() (SecurityKey != SecurityCheck)"); // then we have an invalid matrix.
  120. }
  121. // TokenMatrix::Verify(key)
  122. void TokenMatrix::Verify(string& SecurityKey) { // Builds and verifies a file digest.
  123. MANGLER DigestChecker; // Create a mangler for the digest.
  124. // Gain access to our token matrix as bytes.
  125. unsigned char* TokensAsBytes = reinterpret_cast<unsigned char*>(Matrix);
  126. int BytesInTokenMatrix = (MatrixSize * sizeof(Token));
  127. // Initialize our digest engine with the security key.
  128. int SecurityKeyLength = SecurityKey.length(); // For the length of our key
  129. for(int a=0;a<SecurityKeyLength;a++) // feed each byte through the
  130. DigestChecker.Encrypt(SecurityKey.at(a)); // mangler to evolve the key
  131. // state.
  132. // Build the digest.
  133. int IndexOfDigest = // Find the index of the digest by
  134. BytesInTokenMatrix - // starting at the end of the matrix,
  135. SecurityKeyBufferSize - // backing up past the security key,
  136. RulebaseDigestSize; // then past the digest.
  137. int a=0; // Keep track of where we are.
  138. for(;a<IndexOfDigest;a++) // Loop through up to the digest and
  139. DigestChecker.Encrypt(TokensAsBytes[a]); // pump the file through the mangler.
  140. // Now that the digest is built we must test it.
  141. // The original was emitted by encrypting 0s so if we do the same thing we will match.
  142. for(int b=0;b<RulebaseDigestSize;b++) // Loop through the digest and compare
  143. if(DigestChecker.Encrypt(0)!=TokensAsBytes[a+b]) // our digest to the stored digest. If
  144. throw BadMatrix("TokenMatrix::Verify() Bad Digest"); // any byte doesn't match it's bad!
  145. // If we made it through all of that then we're valid :-)
  146. }
  147. void TokenMatrix::FlipEndian() { // Converts big/little endian tokens.
  148. assert(sizeof(unsigned int)==4); // Check our assumptions.
  149. unsigned int* UInts = reinterpret_cast<unsigned int*>(Matrix); // Grab the matrix as uints.
  150. int Length = ((MatrixSize * sizeof(Token)) / sizeof(unsigned int)); // Calculate it's size.
  151. for(int i = 0; i < Length; i++) { // Loop through the array of u ints
  152. unsigned int x = UInts[i]; // and re-order the bytes in each
  153. x = ((x & 0xff000000) >> 24) | // one to swap from big/little endian
  154. ((x & 0x00ff0000) >> 8) | // to little/big endian.
  155. ((x & 0x0000ff00) << 8) |
  156. ((x & 0x000000ff) << 24);
  157. UInts[i] = x; // Put the flipped int back.
  158. }
  159. }
  160. // Evaluator Implementations //////////////////////////////////////////////////////////////
  161. // 20030216 _M Optimization conversions
  162. inline int Evaluator::i_lower() { return myEvaluationMatrix->i_lower; }
  163. inline bool Evaluator::i_isDigit() { return myEvaluationMatrix->i_isDigit; }
  164. inline bool Evaluator::i_isSpace() { return myEvaluationMatrix->i_isSpace; }
  165. inline bool Evaluator::i_isAlpha() { return myEvaluationMatrix->i_isAlpha; }
  166. // Evaluator::Evaluator(position,evalmatrix) Constructor
  167. Evaluator::Evaluator(int s, EvaluationMatrix* m) { // Constructor...
  168. myEvaluationMatrix = m; // Capture the matrix I live in.
  169. Matrix = myEvaluationMatrix->getTokens(); // Capture the token matrix I walk in.
  170. MatrixSize = myEvaluationMatrix->getMatrixSize(); // And get it's size.
  171. PositionLimit = MatrixSize - 256; // Calculate the safety limit.
  172. StreamStartPosition = s; // Always record our starting point.
  173. NextEvaluator = NULL; // Allways start off with no extensions.
  174. CurrentPosition = 0; // Always start at the root of the matrix;
  175. WildRunLength = 0; // No run length when new.
  176. Condition = DOING_OK; // Start off being ok.
  177. }
  178. // Evaluator::EvaluateThis()
  179. Evaluator::States Evaluator::EvaluateThis(unsigned short int i) { // Follow the this byte.
  180. Condition = FALLEN_OFF; // Start off guessing we'll fall off.
  181. // First upgrade will be to DOING_OK, after that we launch buddies.
  182. // In order to handle wildcard characters, this evaluation function must actually
  183. // compare the character to a number of possibilities in most-specific to least-
  184. // specific order to see if any match. In order to support overlapping rule sets,
  185. // if more than one wildcard matches at this node, an additional evaluator will be
  186. // placed in line already _AT THIS PATH POINT_ so that both possibilities will be
  187. // explored. New evaluators are always added at the TOP of the list so we are always
  188. // guaranteed not to overdrive an evaluator and end up in a recursive race condition.
  189. // 20030216 _M Optimizations. In order to reduce the number of instructions per byte
  190. // the parent Evaluation Matrix will now translate the byte i into boolean flags
  191. // indicating if they are digits, white, letters, etc... and converting to lower
  192. // case etc... This conversion is then done only once so that thereafter only a simple
  193. // comparison need be made. This should eliminate many function calls and a collection
  194. // of numeric comparisons.
  195. //
  196. // I am also moving the simple comparisons to the front of each logical section so
  197. // that failures there can short-circuit subsequent logic to view the state of the
  198. // matrix regardin that character. The matrix lookup is likely to be more expensive
  199. // than a single binary comparison.
  200. // For safety, we check our evaluation position here - If xNoCase is out of range
  201. // then we will return OUT_OF_RANGE to indicate the problem rather than accessing
  202. // data beyone our token matrix's limits.
  203. /*** 20070606 _M Reduced the strength of this check from 3 comparisons to 1.
  204. **** CurrentPosition is now an unsigned int so it cannot be negative. The limit
  205. **** is now calculated once in the constructor as PositionLimit.
  206. if(
  207. CurrentPosition < 0 || // Position should never be < 0
  208. xPrecise >= MatrixSize || // nor xPrecise over the top.
  209. xNoCase >= MatrixSize // nor NoCase over the top.
  210. ) // If either occur we have a
  211. return Condition = OUT_OF_RANGE; // bad matrix.
  212. ***/
  213. if(CurrentPosition >= PositionLimit) return Condition = OUT_OF_RANGE;
  214. // All of the positions calculated below are guaranteed to be within the ranges checked
  215. // above so we're safe if we get to this point.
  216. // So, at this point it's safe to check and see if I'm terminated. Note that if I
  217. // am at a termination point, my path has terminated and I have a symbol so I don't
  218. // need to resolve any more characters - even the current one.
  219. if(Matrix[CurrentPosition].isTermination()) return Condition = TERMINATED;
  220. // NOTE: The above is written for sudden-death termination. Eventually we will want
  221. // to support deep - filters which will show every rule match and this will need to
  222. // be rewritten.
  223. // Evaluation order, most-to-least specific:
  224. int xPrecise = CurrentPosition + i; // Match Precise Character
  225. int xNoCase = CurrentPosition + i_lower(); // Match Case insensitive
  226. // Of course I may need to resolve some of the following
  227. // wildcard characters.
  228. int xLetter = CurrentPosition + WILD_LETTER; // Match Any letter.
  229. int xDigit = CurrentPosition + WILD_DIGIT; // Match Any digit.
  230. int xNonWhite = CurrentPosition + WILD_NONWHITE; // Match Any non-whitespace.
  231. int xWhiteSpace = CurrentPosition + WILD_WHITESPACE; // Match Any whitespace.
  232. int xAnyInline = CurrentPosition + WILD_INLINE; // Match Any byte but new line.
  233. int xAnything = CurrentPosition + WILD_ANYTHING; // Match Any character at all.
  234. int xRunGateway = CurrentPosition + RUN_GATEWAY; // Match the run-loop gateway.
  235. // Try to match the precise character.
  236. if(Matrix[xPrecise].Character() == i) { // If we've matched our path
  237. Condition = DOING_OK; // upgrade to doing ok.
  238. CurrentPosition = xPrecise +
  239. Matrix[xPrecise].Vector; // Move myself along this path.
  240. }
  241. // Try to match the case insensitive character.
  242. if(i_lower()!=i && Matrix[xNoCase].Character()==i_lower()){
  243. // If we've matched our path
  244. // with a compromized case then
  245. if(Condition==FALLEN_OFF) { // check: if no matches yet,
  246. Condition = DOING_OK; // upgrade to doing ok.
  247. CurrentPosition = xNoCase +
  248. Matrix[xNoCase].Vector; // Move myself along this path.
  249. }
  250. // If we more than one match then
  251. else { // lets try to make a buddy...
  252. // If there's no duplicate buddy like this already, then we'll create one.
  253. // To create a buddy, add an evaluator at the top of the list (behind us) and
  254. // set it's position as if it had been here all along and had matched the current
  255. // character. Next time we evaluate it will be just like all the others.
  256. myEvaluationMatrix->
  257. AddEvaluator(StreamStartPosition,Matrix[xNoCase].Vector+xNoCase);
  258. }
  259. }
  260. // Start looking at wildcards... Here's where we must limit run length.
  261. if(Condition == DOING_OK) // If we matched above we'll
  262. WildRunLength = 0; // reset our wild run count.
  263. // If not then we need to keep
  264. else { // track of our run length.
  265. ++WildRunLength; // Count up the run length.
  266. if(WildRunLength >= MaxWildRunLength) // If we exceed the max then
  267. return Condition = FALLEN_OFF; // we've fallen off the path
  268. } // and we do it immediately.
  269. // WILD_LETTER
  270. // If that didn't do it for us...
  271. // Try to match any letter character.
  272. // The way this next one works (and the rest of the wildcards) is we look into
  273. // the token matrix to see if the wildcard is part of the current path... If it
  274. // is then we compare the incoming character to that wildcard evaluation function
  275. // and if it is true, then we've got a match.
  276. if(i_isAlpha() && Matrix[xLetter].Character()==WILD_LETTER){
  277. // If we've matched our path
  278. // with any letter then
  279. if(Condition==FALLEN_OFF) { // check: if no matches yet,
  280. Condition = DOING_OK; // upgrade to doing ok.
  281. CurrentPosition = xLetter +
  282. Matrix[xLetter].Vector; // Move myself along this path.
  283. }
  284. else { // Otherwise make a buddy...
  285. // If there's no duplicate buddy like this already, then we'll create one.
  286. // To create a buddy, add an evaluator at the top of the list (behind us) and
  287. // set it's position as if it had been here all along and had matched the current
  288. // character. Next time we evaluate it will be just like all the others.
  289. myEvaluationMatrix->
  290. AddEvaluator(StreamStartPosition,Matrix[xLetter].Vector+xLetter);
  291. }
  292. }
  293. // WILD_DIGIT
  294. // If that didn't do it for us...
  295. // Try to match any digit character.
  296. if(i_isDigit() && Matrix[xDigit].Character()==WILD_DIGIT){
  297. // If we've matched our path
  298. // with any letter then
  299. if(Condition==FALLEN_OFF) { // check: if no matches yet,
  300. Condition = DOING_OK; // upgrade to doing ok.
  301. CurrentPosition = xDigit +
  302. Matrix[xDigit].Vector; // Move myself along this path.
  303. }
  304. else { // Otherwise make a buddy...
  305. // If there's no duplicate buddy like this already, then we'll create one.
  306. // To create a buddy, add an evaluator at the top of the list (behind us) and
  307. // set it's position as if it had been here all along and had matched the current
  308. // character. Next time we evaluate it will be just like all the others.
  309. myEvaluationMatrix->
  310. AddEvaluator(StreamStartPosition,Matrix[xDigit].Vector+xDigit);
  311. }
  312. }
  313. // WILD_NONWHITE
  314. // If that didn't do it for us...
  315. // Try to match any non-whitespace character.
  316. if(!i_isSpace() && Matrix[xNonWhite].Character()==WILD_NONWHITE){
  317. // If we've matched our path
  318. // with any letter then
  319. if(Condition==FALLEN_OFF) { // check: if no matches yet,
  320. Condition = DOING_OK; // upgrade to doing ok.
  321. CurrentPosition = xNonWhite +
  322. Matrix[xNonWhite].Vector; // Move myself along this path.
  323. }
  324. else { // Otherwise make a buddy...
  325. // If there's no duplicate buddy like this already, then we'll create one.
  326. // To create a buddy, add an evaluator at the top of the list (behind us) and
  327. // set it's position as if it had been here all along and had matched the current
  328. // character. Next time we evaluate it will be just like all the others.
  329. myEvaluationMatrix->
  330. AddEvaluator(StreamStartPosition,Matrix[xNonWhite].Vector+xNonWhite);
  331. }
  332. }
  333. // WILD_WHITESPACE
  334. // If that didn't do it for us...
  335. // Try to match any whitespace character.
  336. if(i_isSpace() && Matrix[xWhiteSpace].Character()==WILD_WHITESPACE){
  337. // If we've matched our path
  338. // with any whitespace then
  339. if(Condition==FALLEN_OFF) { // check: if no matches yet,
  340. Condition = DOING_OK; // upgrade to doing ok.
  341. CurrentPosition = xWhiteSpace +
  342. Matrix[xWhiteSpace].Vector; // Move myself along this path.
  343. }
  344. else { // Otherwise make a buddy...
  345. // If there's no duplicate buddy like this already, then we'll create one.
  346. // To create a buddy, add an evaluator at the top of the list (behind us) and
  347. // set it's position as if it had been here all along and had matched the current
  348. // character. Next time we evaluate it will be just like all the others.
  349. myEvaluationMatrix->
  350. AddEvaluator(StreamStartPosition,Matrix[xWhiteSpace].Vector+xWhiteSpace);
  351. }
  352. }
  353. // WILD_INLINE
  354. // If that didn't do it for us...
  355. // Try to match any character EXCEPT a new line.
  356. if(i != '\n' && Matrix[xAnyInline].Character()==WILD_INLINE){
  357. // If we've matched our path
  358. // with any byte but \n then
  359. if(Condition==FALLEN_OFF) { // check: if no matches yet,
  360. Condition = DOING_OK; // upgrade to doing ok.
  361. CurrentPosition = xAnyInline +
  362. Matrix[xAnyInline].Vector; // Move myself along this path.
  363. }
  364. else { // Otherwise make a buddy...
  365. // If there's no duplicate buddy like this already, then we'll create one.
  366. // To create a buddy, add an evaluator at the top of the list (behind us) and
  367. // set it's position as if it had been here all along and had matched the current
  368. // character. Next time we evaluate it will be just like all the others.
  369. myEvaluationMatrix->
  370. AddEvaluator(StreamStartPosition,Matrix[xAnyInline].Vector+xAnyInline);
  371. }
  372. }
  373. // WILD_ANYTHING
  374. // If that didn't do it for us...
  375. // Try to match any character.
  376. if(Matrix[xAnything].Character()==WILD_ANYTHING){
  377. // If we've matched our path
  378. // with any letter then
  379. if(Condition==FALLEN_OFF) { // check: if no matches yet,
  380. Condition = DOING_OK; // upgrade to doing ok.
  381. CurrentPosition = xAnything +
  382. Matrix[xAnything].Vector; // Move myself along this path.
  383. }
  384. else { // Otherwise make a buddy...
  385. // If there's no duplicate buddy like this already, then we'll create one.
  386. // To create a buddy, add an evaluator at the top of the list (behind us) and
  387. // set it's position as if it had been here all along and had matched the current
  388. // character. Next time we evaluate it will be just like all the others.
  389. myEvaluationMatrix->
  390. AddEvaluator(StreamStartPosition,Matrix[xAnything].Vector+xAnything);
  391. }
  392. }
  393. // 20021112 _M
  394. // Beginning with version 2 of Message Sniffer we've implemented a new construct
  395. // for run-loops that prevents any interference between rules where run-loops might
  396. // appear in locations coinciding with standard match bytes. The new methodology
  397. // uses a special run-loop-gateway character to isolate any run loops from standard
  398. // nodes in the matrix. Whenever a run-loop gateway is present at a node a buddy is
  399. // inserted AFTER the current evaluator so that it will evaluate the current character
  400. // from the position of the run-loop gateway. This allows run loops to occupy the same
  401. // positional space as standard matches while maintaining isolation between their paths
  402. // in the matrix.
  403. // We don't want to launch any run loop buddies unless we matched this far. If we did
  404. // match up to this point and the next character in a pattern includes a run loop then
  405. // we will find a gateway byte at this point representing the path to any run loops.
  406. // If we made it this far launch a buddy for any run-loop gateway that's present.
  407. // Of course, the buddy must be evaluated after this evaluator during this pass because
  408. // he will have shown up late... That is, we don't detect a run gateway until we're
  409. // sitting on a new node looking for a result... The very result we may be looking for
  410. // could be behind the gateway - so we launch the buddy behind us and he will be able
  411. // to match anything in this pass that we missed when looking for a non-run match.
  412. if(Matrix[xRunGateway].Character() == RUN_GATEWAY)
  413. myEvaluationMatrix->
  414. InsEvaluator(StreamStartPosition,Matrix[xRunGateway].Vector+xRunGateway);
  415. // At this point, we've tried all of our rules, and created any buddies we needed.
  416. // If we got a match, we terminated long ago. If we didn't, then we either stayed
  417. // on the path or we fell off. Either way, the flag is in Condition so we can send
  418. // it on.
  419. return Condition;
  420. }
  421. ///////////////////////////////////////////////////////////////////////////////////////////
  422. // EvaluationMatrix Implementations ///////////////////////////////////////////////////////
  423. // EvaluationMatrix::AddMatchRecord(int sp, int ep, int sym)
  424. // Most of this functionality is about deep scans - which have been put on hold for now
  425. // due to the complexity and the scope of the current application. For now, although
  426. // we will use this reporting mechanism, it will generally record only one event.
  427. MatchRecord* EvaluationMatrix::AddMatchRecord(int sp, int ep, int sym) {
  428. // 20030216 _M Added range check code to watch for corruption. Some systems have
  429. // reported matches with zero length indicating an undetected corruption. This
  430. // range check will detect and report it.
  431. if(sp==ep) // Check that we're in range - no zero
  432. throw OutOfRange("sp==ep"); // length pattern matches allowed!
  433. MatchRecord* NewMatchRecord = // Then, create the new result object
  434. new MatchRecord(sp,ep,sym); // by passing it the important parts.
  435. if(NewMatchRecord==NULL) // Check for a bad allocation and throw
  436. throw BadAllocation("NewMatchRecord==NULL"); // an exception if that happens.
  437. if(ResultList == NULL) { // If this is our first result we simply
  438. ResultList = NewMatchRecord; // add the result to our list, and of course
  439. LastResultInList = NewMatchRecord; // it is the end of the list as well.
  440. } else { // If we already have some results, then
  441. LastResultInList->NextMatchRecord = // we add the new record to the result list
  442. NewMatchRecord; // and record that the new record is now the
  443. LastResultInList = NewMatchRecord; // last result in the list.
  444. }
  445. return NewMatchRecord; // Return our new match record.
  446. }
  447. // EvaluationMatrix::AddEvaluator()
  448. // 20021112 _M
  449. // This function has be modified to include a check for duplicates as well as setting
  450. // the mount point for the new evaluator. This eliminates a good deal of code elsewhere
  451. // and encapsulates the complete operation. If a duplicate evaluator is found then the
  452. // function returns NULL indicating that nothing was done. In practic, no check is made
  453. // since any serious error conditions cause errors to be thrown from within this function
  454. // call. These notes apply to some extent to InsEvaluator which is copied from this function
  455. // and which has the only difference of putting the new evaluator after the current one
  456. // in the chain in order to support branch-out operations for loop sequences in the matrix.
  457. Evaluator* EvaluationMatrix::AddEvaluator(int s, int m) { // Adds a new evaluator at top.
  458. if(!isNoDuplicate(m)) return NULL; // If there is a duplicate do nothing.
  459. if(CountOfEvaluators >= MAX_EVALS) // If we've exceeded our population size
  460. throw MaxEvalsExceeded("Add:CountOfEvaluators >= MAX_EVALS"); // then throw an exception.
  461. Evaluator* NewEvaluator = SourceEvaluator(s,this); // Make up a new evaluator.
  462. if(NewEvaluator == NULL) // Check for a bad allocation and throw
  463. throw BadAllocation("Add:NewEvaluator == NULL"); // an exception if it happens.
  464. NewEvaluator->NextEvaluator = EvaluatorList; // Point the new evaluator to the list.
  465. EvaluatorList = NewEvaluator; // Then point the list head to
  466. // the new evaluator.
  467. NewEvaluator->CurrentPosition = m; // Esablish the mount point.
  468. ++CountOfEvaluators; // Add one to our evaluator count.
  469. if(CountOfEvaluators > MaximumCountOfEvaluators) // If the count is the biggest we
  470. MaximumCountOfEvaluators = CountOfEvaluators; // have seen then keep track of it.
  471. return NewEvaluator; // Return the new evaluator.
  472. }
  473. // EvaluationMatrix::InsEvaluator()
  474. Evaluator* EvaluationMatrix::InsEvaluator(int s, int m) { // Inserts a new evaluator.
  475. if(!isNoDuplicate(m)) return NULL; // If there is a duplicate do nothing.
  476. if(CountOfEvaluators >= MAX_EVALS) // If we've exceeded our population size
  477. throw MaxEvalsExceeded("Ins:CountOfEvaluators >= MAX_EVALS"); // then throw an exception.
  478. Evaluator* NewEvaluator = SourceEvaluator(s,this); // Make up a new evaluator.
  479. if(NewEvaluator == NULL) // Check for a bad allocation and throw
  480. throw BadAllocation("Ins:NewEvaluator == NULL"); // an exception if it happens.
  481. NewEvaluator->NextEvaluator = // Point the new evaluator where the
  482. CurrentEvaluator->NextEvaluator; // current evalautor points... then point
  483. CurrentEvaluator->NextEvaluator = // the current evaluator to this one. This
  484. NewEvaluator; // accomplishes the insert operation.
  485. NewEvaluator->CurrentPosition = m; // Esablish the mount point.
  486. ++CountOfEvaluators; // Add one to our evaluator count.
  487. if(CountOfEvaluators > MaximumCountOfEvaluators) // If the count is the biggest we
  488. MaximumCountOfEvaluators = CountOfEvaluators; // have seen then keep track of it.
  489. return NewEvaluator; // Return the new evaluator.
  490. }
  491. // EvaluationMatrix::DropEvaluator()
  492. void EvaluationMatrix::DropEvaluator() { // Drops the current evaluator from the matrix.
  493. Evaluator* WhereTo = CurrentEvaluator->NextEvaluator; // Where do we go from here?
  494. // First step is to heal the list as if the current evaluator were not present.
  495. // If there is no previous evaluator - meaning this should be the first one in the
  496. // list - then we point the list head to the next evaluator on the list (WhereTo)
  497. if(PreviousEvaluator != NULL) // If we have a Previous then
  498. PreviousEvaluator->NextEvaluator = WhereTo; // our next is it's next.
  499. else // If we don't then our next
  500. EvaluatorList = WhereTo; // is the first in the list.
  501. // Now that our list is properly healed, it's time to drop the dead evaluator and
  502. // get on with our lives...
  503. CurrentEvaluator->NextEvaluator = NULL; // Disconnect from any list.
  504. CacheEvaluator(CurrentEvaluator); // Drop the current eval.
  505. CurrentEvaluator = WhereTo; // Move on.
  506. --CountOfEvaluators; // Reduce our evaluator count.
  507. }
  508. // EvaluationMatrix::EvaluateThis()
  509. //
  510. // This function returns the number of matches that were found. It is possible for more
  511. // than one evaluator to match on a single character.
  512. //
  513. // 0 indicates no matches were found.
  514. // >0 indicates some matches were found.
  515. // If there is a problem then an exception will be thrown.
  516. int EvaluationMatrix::EvaluateThis(unsigned short int i) {
  517. AddEvaluator(CountOfCharacters,0); // First, add a new Evaluator at the root of the
  518. // matrix for the current position in the scan
  519. // stream.
  520. // The new evaluator is now at the top of our list.
  521. // If there was a problem then an exception will have been thrown.
  522. // If our allocation worked ok, then we'll be here and ready to start scanning
  523. // the rule set with our current character.
  524. PassResult = 0; // Start by assuming we won't match.
  525. CurrentEvaluator = EvaluatorList; // Start at the top of the list.
  526. PreviousEvaluator = NULL; // NULL means previous is the top.
  527. // 20030216 _M
  528. // Next do some basic conversions and evaluations so they don't need to be done
  529. // again within the evaluators. From now on the evaluators will look here for basic
  530. // conversions and boolean check values rather than performing the checks themselves.
  531. i_lower = tolower(i); // Convert i to lower case.
  532. i_isDigit = isdigit(i); // Check for a digit.
  533. i_isSpace = isspace(i); // Check for whitespace.
  534. i_isAlpha = isalpha(i); // Check for letters.
  535. // Next, loop through the list and pass the incoming character to
  536. // each evaluator. Drop those that fall off, and record those that terminate. The
  537. // rest of them stick around to walk their paths until they meet their fate.
  538. while(CurrentEvaluator != NULL) { // While there are more evaluators...
  539. // go through the list and evaluate
  540. switch(CurrentEvaluator->EvaluateThis(i)) { // the current character against each.
  541. case Evaluator::FALLEN_OFF: { // If we've fallen off the path
  542. DropEvaluator(); // drop the current evaluator and
  543. break; // move on with our lives.
  544. }
  545. case Evaluator::DOING_OK: { // If we're still going then...
  546. PreviousEvaluator = CurrentEvaluator; // keep track of where we've been and
  547. CurrentEvaluator = // move forward to the next evaluator
  548. CurrentEvaluator->NextEvaluator; // in the list.
  549. break;
  550. }
  551. case Evaluator::TERMINATED: { // If we've terminated a path...
  552. ++PassResult; // Record our PassResult.
  553. // Create a new match result using the data in the current evaluator.
  554. // If there is a problem adding the match an exception will be thrown.
  555. AddMatchRecord(
  556. CurrentEvaluator->StreamStartPosition,
  557. CountOfCharacters - 1,
  558. myTokenMatrix->Symbol(CurrentEvaluator->CurrentPosition)
  559. );
  560. // From Version 2 onward we're always doing deep scans...
  561. // Having successfully recorded the result of this critter we can kill them off.
  562. DropEvaluator(); // He's dead.
  563. break; // Now let's keep looking.
  564. }
  565. case Evaluator::OUT_OF_RANGE: { // This result is really bad and
  566. throw OutOfRange("case Evaluator::OUT_OF_RANGE:"); // probably means we have a bad matrix.
  567. break;
  568. // The reason we don't throw OutOfRange from within the evaluator is that we
  569. // may want to take some other action in the future... So, we allow the evaluator
  570. // to tell us we sent it out of range and then we decide what to do about it.
  571. }
  572. }
  573. }
  574. // At the end of this function our PassResult is either an error (which is
  575. // reported immediately), or it is a match condition. We start out by assuming
  576. // there will be no match. If we find one, then we reset that result... so at
  577. // this point, all we need do is report our findings.
  578. ++CountOfCharacters; // Add one to our Character Count statistic.
  579. // Note that from this point on, the index in the stream is one less than the
  580. // CountOfCharacters... for example, if I've evaluated (am evaluating) one character
  581. // the it's index is 0. This will be important when we create any match records.
  582. return PassResult; // When we're finished, return the last known result.
  583. }