Ви не можете вибрати більше 25 тем Теми мають розпочинатися з літери або цифри, можуть містити дефіси (-) і не повинні перевищувати 35 символів.

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821
  1. // GBUdb.cpp
  2. //
  3. // (C) Copyright 2006 - 2009 ARM Research Labs, LLC
  4. // See www.armresearch.com for the copyright terms.
  5. //
  6. // See GBUdb.hpp for details.
  7. #include <iostream>
  8. #include <fstream>
  9. #include <cstring>
  10. #include <unistd.h>
  11. #include "GBUdb.hpp"
  12. namespace cd = codedweller;
  13. //// Handy utilities...
  14. //// GBUdbDataset implementations //////////////////////////////////////////////
  15. GBUdbDataset::~GBUdbDataset() { // Shutdown a dataset.
  16. if(NULL != DataArray) { // If the DataArray was allocated
  17. delete[] DataArray; // be sure to delete it and
  18. DataArray = NULL; // NULL it's pointer.
  19. }
  20. MyArraySize = 0; // For safety set the size to zero
  21. MyFileName = ""; // and "" the name.
  22. }
  23. GBUdbDataset::GBUdbDataset(const char* SetFileName) : // Open/Create a dataset.
  24. DataArray(NULL), // The array pointer starts as NULL.
  25. MyArraySize(0) { // And the size is zero.
  26. FileName(SetFileName); // Set the file name if provided.
  27. if(0 != MyFileName.length() && (0 == access(MyFileName.c_str(),F_OK))) { // If a file name was provided and exists
  28. load(); // then read the file from disk.
  29. } else { // If the file name was not provided
  30. DataArray = new GBUdbRecord[GBUdbDefaultArraySize]; // then allocate a new Array of
  31. MyArraySize = GBUdbDefaultArraySize; // the default size.
  32. DataArray[ixNextFreeNode()].RawData = // The first new node is the one
  33. GBUdbRootNodeOffset + GBUdbRecordsPerNode; // right after the root node.
  34. DataArray[ixMatchListRoot()].RawData = // Once that's up we can use it to
  35. newMatchNodeRoot(); // allocate the first MatchNode.
  36. }
  37. }
  38. GBUdbDataset::GBUdbDataset(GBUdbDataset& Original) : // Copy constructor.
  39. DataArray(NULL), // The array pointer starts as NULL.
  40. MyArraySize(Original.MyArraySize), // Copy the ArraySize
  41. MyFileName(Original.MyFileName) { // Copy the name pointer.
  42. DataArray = new GBUdbRecord[MyArraySize]; // Allocate a new Array.
  43. memcpy(DataArray, Original.DataArray, sizeof(GBUdbRecord) * MyArraySize); // Copy the data wholesale.
  44. }
  45. const char* GBUdbDataset::FileName(const char* NewName) { // (Re) Set the file name.
  46. MyFileName = ""; // Delete any previous file name.
  47. if(NULL != NewName) { // If we've been given a non-null cstring
  48. MyFileName = NewName; // capture it as our file name.
  49. }
  50. return MyFileName.c_str(); // Return our new FileName.
  51. }
  52. //// During the read, it is safe to plow through the array without
  53. //// checking because any unknown entry points to the zero node and
  54. //// all zero node entries point to the zero node. The read-only
  55. //// method does not add new nodes.
  56. GBUdbRecord& GBUdbDataset::readRecord(unsigned int IP) { // Read a record.
  57. IP = remapIP00toFF(IP); // Make the IP safe for consumption.
  58. int a0, a1, a2, a3; // We will break the IP into 4 octets.
  59. unsigned int xIP = IP; // Grab a copy of IP to maniuplate.
  60. const int LowOctetMask = 0x000000FF; // Mask for seeing the low octet.
  61. const int BitsInOneOctet = 8; // Number of bits to shift per octet.
  62. a3 = xIP & LowOctetMask; xIP >>= BitsInOneOctet; // Grab the a3 octet and shift the IP.
  63. a2 = xIP & LowOctetMask; xIP >>= BitsInOneOctet; // Grab the a2 octet and shift the IP.
  64. a1 = xIP & LowOctetMask; xIP >>= BitsInOneOctet; // Grab the a1 octet and shift the IP.
  65. a0 = xIP & LowOctetMask; // Grab the final octet.
  66. GBUdbIndex RecordIndex = GBUdbRootNodeOffset; // Starting at the root node, follow...
  67. RecordIndex = DataArray[RecordIndex + a0].Index(); // Follow the node then
  68. if(isMatch(RecordIndex)) { // Check for a shortcut (match record).
  69. if(isMatch(RecordIndex, IP)) { return MatchedData(RecordIndex); } // If we have an exact match we're done!
  70. else { return SafeUnknownRecord(); } // If we have a mismatch we are lost...
  71. }
  72. RecordIndex = DataArray[RecordIndex + a1].Index(); // Follow the node then
  73. if(isMatch(RecordIndex)) { // Check for a shortcut (match record).
  74. if(isMatch(RecordIndex, IP)) { return MatchedData(RecordIndex); } // If we have an exact match we're done!
  75. else { return SafeUnknownRecord(); } // If we have a mismatch we are lost...
  76. }
  77. RecordIndex = DataArray[RecordIndex + a2].Index(); // Follow the node. No more match checks.
  78. if(isMatch(RecordIndex)) { // Check for a shortcut (match record).
  79. if(isMatch(RecordIndex, IP)) { return MatchedData(RecordIndex); } // If we have an exact match we're done!
  80. else { return SafeUnknownRecord(); } // If we have a mismatch we are lost...
  81. }
  82. return DataArray[RecordIndex + a3]; // Final node has our data :-)
  83. }
  84. //// dropRecord()
  85. //// This code is essentially a hack of the readRecord() code. If it finds
  86. //// the record it will return true, mark the record as GBUdbUnknown, reduce
  87. //// the IP count, and de-allocate the Match record. Records stored in nodes
  88. //// are set to GBUdbUnknown and the node is left in place - otherwise repeated
  89. //// add and drop operations would lead to leaking all nodes into the match
  90. //// record allocation space. (Node allocation is not a linked list ;-)
  91. bool GBUdbDataset::dropRecord(unsigned int IP) { // Drop an IP record.
  92. IP = remapIP00toFF(IP); // Make the IP safe for consumption.
  93. int a0, a1, a2, a3; // We will break the IP into 4 octets.
  94. unsigned int xIP = IP; // Grab a copy of IP to maniuplate.
  95. const int LowOctetMask = 0x000000FF; // Mask for seeing the low octet.
  96. const int BitsInOneOctet = 8; // Number of bits to shift per octet.
  97. a3 = xIP & LowOctetMask; xIP >>= BitsInOneOctet; // Grab the a3 octet and shift the IP.
  98. a2 = xIP & LowOctetMask; xIP >>= BitsInOneOctet; // Grab the a2 octet and shift the IP.
  99. a1 = xIP & LowOctetMask; xIP >>= BitsInOneOctet; // Grab the a1 octet and shift the IP.
  100. a0 = xIP & LowOctetMask; // Grab the final octet.
  101. GBUdbIndex RecordIndex = GBUdbRootNodeOffset; // Starting at the root node, follow...
  102. GBUdbIndex Node0Index = GBUdbRootNodeOffset; // Keep track of our previous nodes.
  103. GBUdbIndex Node1Index = 0; // This node not set yet.
  104. GBUdbIndex Node2Index = 0; // This node not set yet.
  105. GBUdbIndex Node3Index = 0; // This node not set yet.
  106. RecordIndex = DataArray[Node0Index + a0].Index(); // Follow the node then
  107. if(isMatch(RecordIndex)) { // Check for a shortcut (match record).
  108. if(isMatch(RecordIndex, IP)) { // If we have an exact match we proceed:
  109. MatchedData(RecordIndex).RawData = GBUdbUnknown; // Set the data in the match to unknown.
  110. DataArray[Node0Index + a0].Index(GBUdbUnknown); // Remove the reference to the match record.
  111. deleteMatchAt(RecordIndex); // Reclaim the match record for re-use.
  112. decreaseIPCount(); // Reduce the IP count.
  113. return true; // Return that we were successful.
  114. } else { return false; } // If we have a mismatch we cannot delete.
  115. } else { // If this was a Node link then
  116. Node1Index = RecordIndex; // capture the node root and get ready
  117. } // to follow the next node.
  118. RecordIndex = DataArray[Node1Index + a1].Index(); // Follow the node then
  119. if(isMatch(RecordIndex)) { // Check for a shortcut (match record).
  120. if(isMatch(RecordIndex, IP)) { // If we have an exact match we proceed:
  121. MatchedData(RecordIndex).RawData = GBUdbUnknown; // Set the data in the match to unknown.
  122. DataArray[Node1Index + a1].Index(GBUdbUnknown); // Remove the reference to the match record.
  123. deleteMatchAt(RecordIndex); // Reclaim the match record for re-use.
  124. decreaseIPCount(); // Reduce the IP count.
  125. return true; // Return that we were successful.
  126. } else { return false; } // If we have a mismatch we cannot delete.
  127. } else { // If this was a Node link then
  128. Node2Index = RecordIndex; // capture the node root and get ready
  129. } // to follow the next node.
  130. RecordIndex = DataArray[Node2Index + a2].Index(); // Follow the node then
  131. if(isMatch(RecordIndex)) { // Check for a shortcut (match record).
  132. if(isMatch(RecordIndex, IP)) { // If we have an exact match we proceed:
  133. MatchedData(RecordIndex).RawData = GBUdbUnknown; // Set the data in the match to unknown.
  134. DataArray[Node2Index + a2].Index(GBUdbUnknown); // Remove the reference to the match record.
  135. deleteMatchAt(RecordIndex); // Reclaim the match record for re-use.
  136. decreaseIPCount(); // Reduce the IP count.
  137. return true; // Return that we were successful.
  138. } else { return false; } // If we have a mismatch we cannot delete.
  139. } else { // If this was a Node link then
  140. Node3Index = RecordIndex; // capture the node root and get ready
  141. } // to follow the next node.
  142. RecordIndex = Node3Index + a3; // Follow the node.
  143. if(GBUdbUnknown != DataArray[RecordIndex].RawData) { // If there is data there then
  144. DataArray[RecordIndex].RawData = GBUdbUnknown; // mark the entry as unknown,
  145. decreaseIPCount(); // decrease the IP count
  146. return true; // and return true.
  147. } // If we got all the way to the end and
  148. return false; // didn't find a match then return false.
  149. }
  150. /* Ahhh, the simple life. In a single mode lightning index, each key
  151. ** octet lives in a node, so when you grow a new path you either follow
  152. ** existing nodes or make new ones. We're not doing that here, but as
  153. ** a reference here is how that is usually handled:
  154. **
  155. GBUdbIndex GBUdbDataset::invokeAt(GBUdbRecord& R) { // Invoke at Record.
  156. if(GBUdbUnknown == R.RawData) { // If the record does not point to a
  157. R.Index(newNodeRoot()); // node then give it a new node.
  158. } // If the record already has a node
  159. return R.Index(); // or we gave it one, then follow it.
  160. }
  161. */
  162. //// Little helper function for invokeAt()
  163. int getOctet(int Octet, unsigned int IP) { // Returns Octet number Octet from IP.
  164. const int BitsInOneOctet = 8; // Number of bits to shift per octet.
  165. const int LowOctetMask = 0x000000FF; // Mask for seeing the low octet.
  166. int BitsToShift = 0; // Assume we want a3 but
  167. switch(Octet) { // If we don't, use this handy switch.
  168. case 0: { BitsToShift = 3 * BitsInOneOctet; break; } // For octet 0, shift out 3 octets.
  169. case 1: { BitsToShift = 2 * BitsInOneOctet; break; } // For octet 1, shift out 2 octets.
  170. case 2: { BitsToShift = 1 * BitsInOneOctet; break; } // For octet 2, shift out 1 octets.
  171. } // For octet 3, shift none more octets.
  172. if(0 < BitsToShift) { // If we have bits to shift then
  173. IP >>= BitsToShift; // shift them.
  174. }
  175. return (IP & LowOctetMask); // Exctract the octet at the bottom.
  176. }
  177. //// invokeAt() is a helper function that encapsulates the work of growing new
  178. //// pathways. There are several cases to handle in a bimodal indexing scheme
  179. //// since sometimes you extend new nodes (as commented out above), and some-
  180. //// times you create MatchRecords, and sometimes you have collisions and
  181. //// have to extend previous matches.... or not. All of that will become clear
  182. //// shortly ;-) The good news is that at least invokeAt() is always supposed
  183. //// to return the next place to go --- that is, you never get lost because if
  184. //// the next step in the path does not exist yet then you create it.
  185. GBUdbIndex GBUdbDataset::invokeAt(GBUdbRecord& R, unsigned int IP, int Octet, bool ExtendMatches) {
  186. // R is either known (goes somewhere) or unknown (we would be lost).
  187. // IF R is UNNKOWN then we ...
  188. //// create a match and return it. (No conflict, no extension, no extra node :-)
  189. //**** We got out of that one so we're back at the root level.
  190. if(GBUdbUnknown == R.RawData) {
  191. R.Index(newMatchRecord(IP));
  192. return R.Index();
  193. }
  194. // ELSE R is KNOWN then it either points to a MatchRecord or a Node.
  195. //// IF R points to a Node then we will simply follow it.
  196. //**** We got out of that one so we're back at the root level.
  197. if(!isMatch(R.Index())) {
  198. return R.Index();
  199. }
  200. // ELSE R points to a MatchRecord then we get more complex.
  201. //// IF the MatchRecord matches our IP then we simply follow it.
  202. //**** We got out of that one so we're back at the root level.
  203. if(isMatch(R.Index(),IP)) {
  204. return R.Index();
  205. }
  206. // ELSE the MatchRecord does not match then we get more complex again...
  207. //// IF we are Extending Matches then we...
  208. ////// create a new node
  209. ////// push the existing match onto the new node
  210. ////// and create a new match for the new IP on that node.
  211. ////// since we already have the solution we return the new match node index (skip a step).
  212. //**** We got out of that one so we're back at the root level.
  213. if(ExtendMatches) { // If we are extending matches
  214. GBUdbIndex I = newNodeRoot(); // we create a new node.
  215. int NewSlotForCurrentMatch = // Locate the slot in that node where
  216. getOctet( // the current match should reside
  217. Octet + 1, // based on the octet after this one
  218. DataArray[R.Index()] // by extracting that octet from
  219. .RawData); // the MatchReord header.
  220. // Then we put the current match into
  221. DataArray[I + NewSlotForCurrentMatch].Index(R.Index()); // the correct slot on the new node,
  222. return R.Index(I); // point the current slot to that node
  223. } // and return the node to be followed.
  224. // ELSE we are NOT Extending Matches then we...
  225. // ** KNOW that we are adding node a3 and dealing with the final octet **
  226. //// create a new node
  227. //// map the existing match data into the new node.
  228. //// delete the existing match (for reallocation). deleteMatchAt(GBUdbIndex I)
  229. //// map the new IP into the new node.
  230. GBUdbIndex I = newNodeRoot(); // Create a new node.
  231. int NewSlotForCurrentMatch = // Locate the slot in that node where
  232. getOctet( // the current match should reside
  233. Octet + 1, // based on the octet after this one
  234. DataArray[R.Index()] // by extracting that octet from
  235. .RawData); // the MatchReord header.
  236. if(ExtendMatches) { // If we are extending matches...
  237. // then we put the current match into
  238. DataArray[I + NewSlotForCurrentMatch].Index(R.Index()); // the correct slot on the new node.
  239. } else { // If we are not extending matches...
  240. // then we must be at the end node so
  241. DataArray[I + NewSlotForCurrentMatch].RawData = // we copy in the data from
  242. MatchedData(R.Index()).RawData; // the current MatchRecord,
  243. deleteMatchAt(R.Index()); // and return the MatchRecord for re-use.
  244. }
  245. return R.Index(I); // Point the current slot to new node
  246. } // and return that node index to follow.
  247. //// The "invoke" method creates all of the needed nodes starting
  248. //// at any point where an "unwknown" entry is found.
  249. GBUdbRecord& GBUdbDataset::invokeRecord(unsigned int IP) { // Invoke a record.
  250. if(FreeNodes() < GBUdbGrowthThreshold) grow(); // If we need more space, make more.
  251. IP = remapIP00toFF(IP); // Make the IP safe for consumption.
  252. int a0, a1, a2, a3; // We will break the IP into 4 octets.
  253. unsigned int xIP = IP; // Grab a copy of IP to maniuplate.
  254. const int LowOctetMask = 0x000000FF; // Mask for seeing the low octet.
  255. const bool Extend = true; // Magic number for extending Matches.
  256. const bool DoNotExtend = false; // Magic number for NOT extending them.
  257. const int BitsInOneOctet = 8; // Number of bits to shift per octet.
  258. a3 = xIP & LowOctetMask; xIP >>= BitsInOneOctet; // Grab the a3 octet and shift the IP.
  259. a2 = xIP & LowOctetMask; xIP >>= BitsInOneOctet; // Grab the a2 octet and shift the IP.
  260. a1 = xIP & LowOctetMask; xIP >>= BitsInOneOctet; // Grab the a1 octet and shift the IP.
  261. a0 = xIP & LowOctetMask; // Grab the final octet.
  262. GBUdbIndex RecordIndex = GBUdbRootNodeOffset; // Starting at the root node,
  263. RecordIndex = invokeAt(DataArray[RecordIndex + a0], IP, 0, Extend); // Invoke w/ possible match outcome.
  264. if(isMatch(RecordIndex, IP)) { // If this resulted in a match
  265. GBUdbRecord& Result = MatchedData(RecordIndex); // then we will grab the match data
  266. increaseIPCountIfNew(Result); // and increase the IP count if it's new.
  267. return Result; // Then we return the result. Done!
  268. }
  269. RecordIndex = invokeAt(DataArray[RecordIndex + a1], IP, 1, Extend); // Invode w/ possible match outcome.
  270. if(isMatch(RecordIndex, IP)) { // If this resulted in a match
  271. GBUdbRecord& Result = MatchedData(RecordIndex); // then we will grab the match data
  272. increaseIPCountIfNew(Result); // and increase the IP count if it's new.
  273. return Result; // Then we return the result. Done!
  274. }
  275. RecordIndex = invokeAt(DataArray[RecordIndex + a2], IP, 2, DoNotExtend); // Invode w/ possible match outcome.
  276. if(isMatch(RecordIndex, IP)) { // If this resulted in a match
  277. GBUdbRecord& Result = MatchedData(RecordIndex); // then we will grab the match data
  278. increaseIPCountIfNew(Result); // and increase the IP count if it's new.
  279. return Result; // Then we return the result. Done!
  280. }
  281. GBUdbRecord& Result = DataArray[RecordIndex + a3]; // Grab the record at the final node.
  282. increaseIPCountIfNew(Result); // If new, increase the IP count.
  283. return Result; // Return the record.
  284. }
  285. void GBUdbDataset::save() { // Flush the GBUdb to disk.
  286. string TempFileName = MyFileName + ".tmp"; // Calculate temp and
  287. string BackFileName = MyFileName + ".bak"; // backup file names.
  288. ofstream dbFile; // Grab a file for writing.
  289. dbFile.open(TempFileName.c_str(), ios::out | ios::binary | ios::trunc); // Open the file and truncate if present.
  290. dbFile.write((char*)DataArray, sizeof(GBUdbRecord) * MyArraySize); // Write our array into the file.
  291. bool AllOK = dbFile.good(); // Are we happy with this?
  292. dbFile.close(); // Close the file when done to be nice.
  293. if(AllOK) { // If everything appears to be ok
  294. unlink(BackFileName.c_str()); // Delete any old backup file we have
  295. rename(MyFileName.c_str(), BackFileName.c_str()); // and make the current file a backup.
  296. rename(TempFileName.c_str(), MyFileName.c_str()); // Then make our new file current.
  297. }
  298. }
  299. const cd::RuntimeCheck SaneFileSizeCheck("GBUdbDataset::load():SaneFileSizeCheck(SaneGBUdbFileSizeLimit <= FileSize)");
  300. void GBUdbDataset::load() { // Read the GBUdb from disk.
  301. ifstream dbFile; // Grab a file for reading.
  302. dbFile.open(MyFileName.c_str(), ios::in | ios::binary); // Open the file with the name we have.
  303. dbFile.seekg(0, ios::end); // Go to the end of the
  304. int FileSize = dbFile.tellg(); // file and back so we can
  305. dbFile.seekg(0, ios::beg); // determine it's size.
  306. int SaneGBUdbFileSizeLimit = (GBUdbDefaultArraySize * sizeof(GBUdbRecord)); // What is a sane size limit?
  307. SaneFileSizeCheck(SaneGBUdbFileSizeLimit <= FileSize); // File size sanity check.
  308. int NewArraySize = FileSize / sizeof(GBUdbRecord); // How many records in this file?
  309. if(NULL != DataArray) { // If we have an array loaded then
  310. delete[] DataArray; // delete the array,
  311. DataArray = NULL; // NULL it's pointer,
  312. MyArraySize = 0; // and zero it's size.
  313. }
  314. DataArray = new GBUdbRecord[NewArraySize]; // Allocate an array of the proper size
  315. MyArraySize = NewArraySize; // set the local size variable
  316. dbFile.read((char*)DataArray,FileSize); // and read the file into the array.
  317. dbFile.close(); // Close when done to be nice.
  318. }
  319. void GBUdbDataset::grow(int HowManyNodes) { // Grow the DataArray.
  320. int NewArraySize = MyArraySize + (HowManyNodes * GBUdbRecordsPerNode); // Calcualte the new array size.
  321. GBUdbRecord* NewDataArray = new GBUdbRecord[NewArraySize]; // Allocate the new array.
  322. int OldArrayLessControl = MyArraySize + GBUdbControlNodeOffset; // Include all records but no control.
  323. memcpy(NewDataArray, DataArray, sizeof(GBUdbRecord) * OldArrayLessControl); // Copy the old data to the new array.
  324. for( // Loop through the control nodes...
  325. int o = MyArraySize + GBUdbControlNodeOffset, // o = old node index
  326. n = NewArraySize + GBUdbControlNodeOffset, // n = new node index
  327. c = GBUdbRecordsPerNode; // c = the record count (how many to do).
  328. c > 0; // For until we run out of records,
  329. c--) { // decrementing the count each time,
  330. NewDataArray[n].RawData = DataArray[o].RawData;n++;o++; // Copy the old control data.
  331. }
  332. delete[] DataArray; // Delete the old data array.
  333. DataArray = NewDataArray; // Swap in the new data array.
  334. MyArraySize = NewArraySize; // Correct the size value.
  335. }
  336. GBUdbIndex GBUdbDataset::newMatchRecord(unsigned int IP) { // Allocate a new Match record for IP.
  337. GBUdbIndex I = DataArray[ixMatchListRoot()].RawData; // Grab the root unused Match Record index.
  338. GBUdbRecord& R = DataArray[I]; // Grab the record itself and inspect it.
  339. if((R.RawData & GBUdbFlagsMask) != GBUdbMatchUnusedBit) { // Check that this looks like an
  340. throw MatchAllocationCorrupted(); // unused match record and if not throw!
  341. } // If all is well then lets proceed.
  342. //// First, let's heal the linked list for future allocations.
  343. if(GBUdbMatchUnusedBit == R.RawData) { // If the match record we are on is
  344. DataArray[ixMatchListRoot()].RawData = // the last in the list then allocate
  345. newMatchNodeRoot(); // a new MatchListNode for the next
  346. } else { // allocation. However, if there are
  347. DataArray[ixMatchListRoot()].RawData = // more records left in the list then
  348. (R.RawData & GBUdbMatchDataMask); // set up the next node for the next
  349. } // allocation.
  350. //// Once that's done we can use the record we have for real data.
  351. R.RawData = EncodedMatch(IP); // Encode the match record for the IP.
  352. return I; // Return the match record's index.
  353. }
  354. GBUdbIndex GBUdbDataset::newMatchNodeRoot() { // Allocate a new Match node.
  355. GBUdbIndex I = newNodeRoot(); // Grab a new node to convert.
  356. int iLastMatch = GBUdbRecordsPerNode - 2; // Calc the localized i for last match.
  357. for(int i = 0; i < iLastMatch; i+=2) { // Loop through the node
  358. DataArray[I+i].RawData = GBUdbMatchUnusedBit | (I+i+2); // Build a linked list of Unused Match
  359. DataArray[I+i+1].RawData = GBUdbUnknown; // records with empty data.
  360. }
  361. DataArray[I+iLastMatch].RawData = GBUdbMatchUnusedBit; // The last record gets a NULL index
  362. DataArray[I+iLastMatch+1].RawData = GBUdbUnknown; // and null data to terminate the list.
  363. return I; // Return the root index.
  364. }
  365. // doForAllRecords()
  366. // This method uses a recursive call to doAllAtNode()
  367. // doAllAtNode sweeps through each record in a node and processes any
  368. // node entries through the next level (calling itself) or directly if
  369. // the node is node3, or if it's pointing to a match record.
  370. void GBUdbDataset::updateWorkingIP(unsigned int& WIP, int OctetValue, int Level) { // Update the Working IP (WIP) at octet Level
  371. switch(Level) {
  372. case 0: { // For the node zero address,
  373. WIP = WIP & 0x00FFFFFF; // Mask out the node zero bits.
  374. OctetValue = OctetValue << 24; // Shift the octet value into position.
  375. WIP = WIP | OctetValue; // Or the octet value bits into place.
  376. break;
  377. }
  378. case 1: {
  379. WIP = WIP & 0xFF00FFFF; // Mask out the node zero bits.
  380. OctetValue = OctetValue << 16; // Shift the octet value into position.
  381. WIP = WIP | OctetValue; // Or the octet value bits into place.
  382. break;
  383. }
  384. case 2: {
  385. WIP = WIP & 0xFFFF00FF; // Mask out the node zero bits.
  386. OctetValue = OctetValue << 8; // Shift the octet value into position.
  387. WIP = WIP | OctetValue; // Or the octet value bits into place.
  388. break;
  389. }
  390. case 3: {
  391. WIP = WIP & 0xFFFFFF00; // Mask out the node zero bits.
  392. WIP = WIP | OctetValue; // Or the octet value bits into place.
  393. break;
  394. }
  395. }
  396. }
  397. //// Note about doAllAtNode(). The x.x.x.0 address is skipped on purpose. This
  398. //// is because all x.x.x.0 addresses are mapped to x.x.x.255. By skipping this
  399. //// address and starting at x.x.x.1 in any search, we do not need to check for
  400. //// x.x.x.0 ips that were remapped. They will simply appear at x.x.x.255.
  401. void GBUdbDataset::doAllAtNode( // Recursively call O with all valid records.
  402. GBUdbIndex I, // Input the node index.
  403. GBUdbOperator& O, // Input the Operator to call.
  404. int NodeLevel, // Input the NodeLevel.
  405. unsigned int WIP // Input the working IP.
  406. ) {
  407. int FirstI = (3 > NodeLevel) ? 0 : 1; // Skip any x.x.x.0 addresses.
  408. for(int i = FirstI; i < GBUdbRecordsPerNode; i++) { // Loop through the slots in this node.
  409. GBUdbIndex RecordIndex = DataArray[I + i].Index(); // Get the record index for this slot.
  410. if(GBUdbUnknown != RecordIndex) { // Check that this slot is not empty.
  411. updateWorkingIP(WIP, i, NodeLevel); // If we've got something then update the WIP.
  412. if(3 > NodeLevel) { // If we are working in rootward nodes:
  413. if(isMatch(RecordIndex)) { // Check for a match record. If we have one then
  414. unsigned int MatchIP = WIP & 0xFF000000; // build the IP for the match from the root
  415. MatchIP |= (DataArray[RecordIndex].RawData & 0x00FFFFFF); // of the WIP and the match IP data.
  416. O(MatchIP, MatchedData(RecordIndex)); // Then call the operator with the matched data.
  417. // If this slot is not a match record
  418. } else { // then it is a node address so we will
  419. doAllAtNode(RecordIndex, O, NodeLevel+1, WIP); // recurse to that node at a deeper level.
  420. }
  421. } else { // If we are working in the last node then
  422. O(WIP, DataArray[I + i]); // call the Operator with this IP & Record.
  423. } // All known data values in the last node are
  424. } // actual data records after all.
  425. }
  426. }
  427. void GBUdbDataset::doForAllRecords(GBUdbOperator& O) { // Call O for every valid record.
  428. unsigned int WorkingIP = 0; // A working IP for all levels to use.
  429. int NodeLevel = 0; // The Node level where we start.
  430. doAllAtNode(GBUdbRootNodeOffset, O, NodeLevel, WorkingIP); // Start at the root node, level 0.
  431. }
  432. //// GBUdb Implementations /////////////////////////////////////////////////////
  433. bool AlertFor(int count) { // True if an alert is needed.
  434. return ( // We want an alert whenever a count
  435. 0x00000001 == count || // hits any of these thresholds. Each
  436. 0x00000002 == count || // threshold is a new bit position
  437. 0x00000004 == count || // indicating that the count has
  438. 0x00000008 == count || // achieved a new power of 2. This
  439. 0x00000010 == count || // mechanism insures that newer IPs
  440. 0x00000020 == count || // get lots of attention while long
  441. 0x00000040 == count || // standing IPs still get visited
  442. 0x00000080 == count || // from time to time as their activity
  443. 0x00000100 == count || // continues.
  444. 0x00000200 == count ||
  445. 0x00000400 == count ||
  446. 0x00000800 == count ||
  447. 0x00001000 == count ||
  448. 0x00002000 == count ||
  449. 0x00004000 == count
  450. );
  451. }
  452. cd::RuntimeCheck GoodTimestampLength("GBUdb.cpp:getTimestamp snprintf(...) == CorrectTimestampLength");
  453. char* getTimestamp(char* TimestampBfr) { // Creates an ISO GMT timestamp.
  454. time_t rawtime; // Get a timer and
  455. tm * gmt; // a time structure.
  456. time(&rawtime); // Grab the current time and
  457. gmt=gmtime(&rawtime); // convert it to GMT.
  458. size_t l = snprintf(TimestampBfr,UTCBufferSize, "%04d%02d%02d%02d%02d%02d", // Format yyyymmddhhmmss
  459. gmt->tm_year+1900,
  460. gmt->tm_mon+1,
  461. gmt->tm_mday,
  462. gmt->tm_hour,
  463. gmt->tm_min,
  464. gmt->tm_sec
  465. );
  466. const size_t CorrectTimestampLength = 4+2+2+2+2+2;
  467. GoodTimestampLength(l == CorrectTimestampLength);
  468. return TimestampBfr;
  469. }
  470. char* getIPString(unsigned int IP, char* bfr) { // Converts an IP to a string.
  471. int a0, a1, a2, a3; // We will break the IP into 4 octets.
  472. const int LowOctetMask = 0x000000FF; // Mask for seeing the low octet.
  473. const int BitsInOneOctet = 8; // Number of bits to shift per octet.
  474. a3 = IP & LowOctetMask; IP >>= BitsInOneOctet; // Grab the a3 octet and shift the IP.
  475. a2 = IP & LowOctetMask; IP >>= BitsInOneOctet; // Grab the a2 octet and shift the IP.
  476. a1 = IP & LowOctetMask; IP >>= BitsInOneOctet; // Grab the a1 octet and shift the IP.
  477. a0 = IP & LowOctetMask; // Grab the final octet.
  478. sprintf(bfr,"%d.%d.%d.%d",a0,a1,a2,a3);
  479. return bfr;
  480. }
  481. void GBUdb::recordAlertFor(unsigned int IP, GBUdbRecord& R, unsigned int C) { // Record an alert event for R if needed.
  482. if(AlertFor(C)) { // If an alert is needed at this level...
  483. GBUdbAlert NewAlert; // Create a new alert record.
  484. NewAlert.IP = IP; // Assign the IP.
  485. NewAlert.R = R; // Assign the Record.
  486. ScopeMutex JustMe(AlertsMutex); // Lock the alerts list mutex.
  487. MyAlerts.push_back(NewAlert); // Add our new alert to the list.
  488. }
  489. }
  490. GBUdbAlert::GBUdbAlert() : // Default constructor gets timestamp.
  491. IP(0) { // IP to zero, R will init to zero
  492. getTimestamp(UTC); // on it's own... Get timestamp.
  493. }
  494. string GBUdbAlert::toXML() { // Convert this alert to XML text
  495. stringstream Alert; // We'll use a stringstream.
  496. const char* FlagName = "ERROR"; // We will want the Flag as text.
  497. switch(R.Flag()) { // Switch on the Flag() value.
  498. case Good: { FlagName = "Good"; break; } // Convert each value to it's name.
  499. case Bad: { FlagName = "Bad"; break; }
  500. case Ugly: { FlagName = "Ugly"; break; }
  501. case Ignore: { FlagName = "Ignore"; break; }
  502. }
  503. char IPStringBfr[20]; // We need a buffer for our IP.
  504. Alert
  505. << "<gbu time=\'" << UTC // GBU alert + timestamp followed
  506. << "\' ip=\'" << getIPString(IP,IPStringBfr) // with the IP,
  507. << "\' t=\'" << FlagName // the type flag,
  508. << "\' b=\'" << R.Bad() // the bad count,
  509. << "\' g=\'" << R.Good() // and the good count.
  510. << "\'/>"; // That's the end.
  511. return Alert.str(); // Return the string.
  512. }
  513. //// Alert import and export - for sharing data between nodes.
  514. void GBUdb::GetAlerts(list<GBUdbAlert>& ListToFill) { // Get all current alerts & clear;
  515. ListToFill.clear(); // Clear out the list to fill.
  516. ScopeMutex JustMe(AlertsMutex); // Lock for a moment.
  517. ListToFill = MyAlerts; // Copy our alerts to the new list.
  518. MyAlerts.clear(); // Clear our alerts.
  519. }
  520. // In order to allow gbudb nodes to interact without swamping their individuality,
  521. // the default mode for integrating thier data is to represent the remote peer's
  522. // influence on a logarithmic scale.
  523. unsigned int rescaleGBUdbCount(unsigned int C) { // Rescale count C for integration.
  524. if(C < 0x00000001) { return 0; } else // Log2, really, .. the short way.
  525. if(C < 0x00000002) { return 1; } else // How many significant bits are in
  526. if(C < 0x00000004) { return 2; } else // the number. Put another way, what
  527. if(C < 0x00000008) { return 3; } else // power of 2 is required to for
  528. if(C < 0x00000010) { return 4; } else // this number.
  529. if(C < 0x00000020) { return 5; } else
  530. if(C < 0x00000040) { return 6; } else
  531. if(C < 0x00000080) { return 7; } else
  532. if(C < 0x00000100) { return 8; } else
  533. if(C < 0x00000200) { return 9; } else
  534. if(C < 0x00000400) { return 10; } else
  535. if(C < 0x00000800) { return 11; } else
  536. if(C < 0x00001000) { return 12; } else
  537. if(C < 0x00002000) { return 13; } else
  538. if(C < 0x00004000) { return 14; } else
  539. return 15;
  540. }
  541. void GBUdb::ImportAlerts(list<GBUdbAlert>& PeerAlerts) { // Integrate peer alerts using log2.
  542. list<GBUdbAlert>::iterator iA;
  543. for(iA = PeerAlerts.begin(); iA != PeerAlerts.end(); iA++) { // Go through the list of PeerAlerts.
  544. GBUdbRecord R = (*iA).R; // Grab the Record in this alert.
  545. R.Bad(rescaleGBUdbCount(R.Bad())); // Adjust the bad and good counts
  546. R.Good(rescaleGBUdbCount(R.Good())); // for integration.
  547. adjustCounts((*iA).IP, R); // Adjust the local counts w/ R.
  548. }
  549. }
  550. //// doForAllRecords
  551. //// This method handles GBUdbOperators and their locking semantics.
  552. //// For full dataset locking the mutex is acquired before calling the
  553. //// dataset's doForAllRecords(). For record locking, the O passed to
  554. //// this method is wrapped in a record locking shim (below) and that is
  555. //// passed to the dataset. If None is selected then the Operator is
  556. //// passed to the dataset as is -- assuming that the Operator will handle
  557. //// it's own locking as needed.
  558. class GBUdbRecordLockingShim : public GBUdbOperator { // Record locking shim for doForAllRecords.
  559. private:
  560. GBUdbOperator& MyOperator; // Reference the Operator we will be servicing.
  561. Mutex& MyMutex; // Reference the Mutex for the GBUdb we are in.
  562. public:
  563. GBUdbRecordLockingShim(GBUdbOperator& O, Mutex& M) : // On construction we grab our critical pieces.
  564. MyOperator(O),
  565. MyMutex(M) {
  566. }
  567. GBUdbRecord& operator()(unsigned int IP, GBUdbRecord& R) { // When our operator() is called
  568. ScopeMutex JustMe(MyMutex); // we lock the mutex in scope and
  569. return MyOperator(IP, R); // call the Operator we're servicing.
  570. } // When we leave scope we unlock (see above).
  571. };
  572. void GBUdb::doForAllRecords(GBUdbOperator& O, GBUdbLocking L) { // Calls O(IP, Record) w/Every record.
  573. if(Dataset == L) { // If we are locking for the Dataset, then
  574. ScopeMutex JustMe(MyMutex); // we will lock the mutex during this
  575. MyDataset->doForAllRecords(O); // entire operation.
  576. } else
  577. if(Record == L) { // If we are locking per record then
  578. GBUdbRecordLockingShim X(O, MyMutex); // we create a record locking shim instance
  579. MyDataset->doForAllRecords(X); // and call O() through that.
  580. } else { // If locking is NOT enabled, then
  581. MyDataset->doForAllRecords(O); // we will call O() without any locking.
  582. }
  583. }
  584. //// The saveSnapshot() method allows us to save a snapshot of our dataset
  585. //// while keeping the mutex locked for as short a time as possible: Just long
  586. //// enough to make a copy of the dataset in RAM.
  587. void GBUdb::saveSnapshot() { // Saves a snapshot of the current db.
  588. GBUdbDataset* Snapshot = NULL; // We need a pointer for our snapshot.
  589. if(NULL == MyDataset) { // If we do not have a dataset to copy
  590. return; // then we simply return.
  591. } else { // If we do have a Dataset to copy...
  592. ScopeMutex JustMe(MyMutex); // Lock the mutex and
  593. Snapshot = new GBUdbDataset(*MyDataset); // make a copy in memory.
  594. } // Then we can unlock the mutex.
  595. Snapshot->save(); // Then outside the mutex we can save.
  596. delete Snapshot; // Once saved we can delete the snapshot.
  597. PostsCounter = 0; // Reset the posts counter.
  598. }
  599. //// reduce()
  600. //// Using the doForAllRecords() functionality, this method reduces all counts
  601. //// by 2 thus renormalizing all records at lower count values. Unknown flagged
  602. //// records who's counts drop to zero will achieve the state GBUdbUnknown. As
  603. //// such, those values would not be carried over in a compress() operation.
  604. class ReduceAll : public GBUdbOperator { // To reduce the good and bad counts.
  605. public:
  606. GBUdbRecord& operator()(unsigned int IP, GBUdbRecord& R) { // Given each record,
  607. R.Good(R.Good() >> 1); // Reduce the Good count by half.
  608. R.Bad(R.Bad() >> 1); // Reduce the Bad count by half.
  609. return R; // Return the record.
  610. }
  611. } ReduceAllOperator;
  612. void GBUdb::reduce() { // Reduce all counts by half.
  613. doForAllRecords(ReduceAllOperator); // Call do for all records with the
  614. } // ReduceAllOperator.
  615. //// compress()
  616. //// Using the doForAllRecords() functionality, this method creates a temporary
  617. //// dataset, copies the existing data into that dataset except where the data
  618. //// is GBUdbUnknown, and then swaps the new dataset in place of the old.
  619. class CompressAll : public GBUdbOperator {
  620. private:
  621. GBUdbDataset* MyOldDataset; // Where do we find the old dataset.
  622. GBUdbDataset* MyNewDataset; // Where do we store our new dataset.
  623. int CountConverted;
  624. int CountDropped;
  625. public:
  626. // Note - There is no destructor. It is expected that the calling function
  627. // will extract the NewDataset and replace the OldDataset when the operation
  628. // has been successful.
  629. CompressAll(GBUdbDataset* OldDataset) : // Startup by
  630. MyOldDataset(OldDataset), // Grabbing the old dataset,
  631. MyNewDataset(NULL), // The new one isn't there yet.
  632. CountConverted(0), // Converted and Dropped
  633. CountDropped(0) { // Counts are zero.
  634. MyNewDataset = new GBUdbDataset(NULL); // Allocate a new Dataset.
  635. MyNewDataset->FileName(OldDataset->FileName()); // Set it's name the same as the old.
  636. } // We don't want to Load() it that way ;-)
  637. GBUdbRecord& operator()(unsigned int IP, GBUdbRecord& R) { // The ForAll Operator goes like this...
  638. if(GBUdbUnknown != R.RawData) { // If the record is not GBUdbUnknown then
  639. MyNewDataset->invokeRecord(IP).RawData = R.RawData; // invoke it and copy it's data.
  640. ++CountConverted; // Increment the converted count.
  641. } else { // If the record is GBUdbUnknown then
  642. ++CountDropped; // count it as dropped and forget it.
  643. }
  644. return R; // Return the record reference.
  645. }
  646. GBUdbDataset* Old() {return MyOldDataset;} // Here we can get our OldDataset pointer.
  647. GBUdbDataset* New() {return MyNewDataset;} // Here we can get our NewDataset pointer.
  648. int Converted() {return CountConverted;} // Here we can get the converted count.
  649. int Dropped() {return CountDropped;} // Here we can get the dropped count.
  650. };
  651. void GBUdb::compress() { // Remove any unknown records (reduced to zero).
  652. CompressAll BuildCompressedDataset(MyDataset); // Create a CompressAll operator for this dataset.
  653. ScopeMutex Freeze(MyMutex); // Lock the mutex for the rest of this operation.
  654. MyDataset->doForAllRecords(BuildCompressedDataset); // Copy all of the active data records.
  655. MyDataset = BuildCompressedDataset.New(); // Put the new dataset in place.
  656. delete BuildCompressedDataset.Old(); // Delete the old dataset.
  657. } // All done, so we're unlocked.
  658. int GBUdb::readIgnoreList(const char* FileName) { // setIgnore for a list of IPs
  659. int IPCount = 0; // Keep track of the IPs we read.
  660. try { // Capture any exceptions.
  661. char IPLineBuffer[256]; // Create a line buffer.
  662. const int SafeBufferSize = sizeof(IPLineBuffer) - 1; // Safe size always leaves a NULL on the end.
  663. ifstream ListFile(FileName, ios::in); // Open up the list file.
  664. while(ListFile.good()) { // While we've got a good file (not eof)
  665. memset(IPLineBuffer, 0, sizeof(IPLineBuffer)); // Clear the buffer.
  666. ListFile.getline(IPLineBuffer, SafeBufferSize); // Read the line. (safely NULL terminated)
  667. // Now we have an IP on a line (in theory). We will parse
  668. // the ip and process any that parse correctly.
  669. // First eat anything that's not a digit.
  670. unsigned long IP = 0L; // We need an IP buffer.
  671. char* cursor = IPLineBuffer; // Start on the first byte.
  672. if('#' == *cursor) continue; // Lines that start with # are comments.
  673. while(0 < *cursor && isspace(*cursor)) ++cursor; // Eat any leading spaces.
  674. // First octet.
  675. if(!isdigit(*cursor)) continue; // If it's not a digit skip this line.
  676. if(255 < atoi(cursor)) continue; // If the octet is out of range skip!
  677. IP += atoi(cursor); IP <<= 8; // Grab the first int and shift it.
  678. while(isdigit(*cursor)) ++cursor; // Eat those digits.
  679. if('.'!=(*cursor)) continue; // If we don't find a dot skip this line.
  680. ++cursor; // If we do, skip the dot.
  681. // Second octet.
  682. if(!isdigit(*cursor)) continue; // If we're not at digit skip this line.
  683. if(255 < atoi(cursor)) continue; // If the octet is out of range skip!
  684. IP += atoi(cursor); IP <<= 8; // Grab the octet and shift things left.
  685. while(isdigit(*cursor)) ++cursor; // Eat those digits.
  686. if('.'!=(*cursor)) continue; // If we don't find a dot skip this line.
  687. ++cursor; // If we do, skip the dot.
  688. // Third octet.
  689. if(!isdigit(*cursor)) continue; // If we're not at digit skip this line.
  690. if(255 < atoi(cursor)) continue; // If the octet is out of range skip!
  691. IP += atoi(cursor); IP <<= 8; // Grab the octet and shift things left.
  692. while(isdigit(*cursor)) ++cursor; // Eat those digits.
  693. if('.'!=(*cursor)) continue; // If we don't find a dot skip this line.
  694. ++cursor; // If we do, skip the dot.
  695. // Last octet.
  696. if(!isdigit(*cursor)) continue; // If we're not at a digit skip this line.
  697. if(255 < atoi(cursor)) continue; // If the octet is out of range skip!
  698. IP += atoi(cursor); // Grab the octet. IP finished!
  699. setIgnore(IP); // Set the IP to Ignore.
  700. ++IPCount; // Bump the IP count.
  701. }
  702. ListFile.close();
  703. }
  704. catch(...) { } // If we have an exception we stop.
  705. return IPCount; // Always return the number of lines read.
  706. }