Du kannst nicht mehr als 25 Themen auswählen Themen müssen mit entweder einem Buchstaben oder einer Ziffer beginnen. Sie können Bindestriche („-“) enthalten und bis zu 35 Zeichen lang sein.

GBUdb.cpp 60KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814
  1. // GBUdb.cpp
  2. //
  3. // (C) Copyright 2006 - 2009 ARM Research Labs, LLC
  4. // See www.armresearch.com for the copyright terms.
  5. //
  6. // See GBUdb.hpp for details.
  7. #include <iostream>
  8. #include <fstream>
  9. #include <cstring>
  10. #include <unistd.h>
  11. #include "GBUdb.hpp"
  12. using namespace std;
  13. //// Handy utilities...
  14. //// GBUdbDataset implementations //////////////////////////////////////////////
  15. GBUdbDataset::~GBUdbDataset() { // Shutdown a dataset.
  16. if(NULL != DataArray) { // If the DataArray was allocated
  17. delete[] DataArray; // be sure to delete it and
  18. DataArray = NULL; // NULL it's pointer.
  19. }
  20. MyArraySize = 0; // For safety set the size to zero
  21. MyFileName = ""; // and "" the name.
  22. }
  23. GBUdbDataset::GBUdbDataset(const char* SetFileName) : // Open/Create a dataset.
  24. DataArray(NULL), // The array pointer starts as NULL.
  25. MyArraySize(0) { // And the size is zero.
  26. FileName(SetFileName); // Set the file name if provided.
  27. if(0 != MyFileName.length() && (0 == access(MyFileName.c_str(),F_OK))) { // If a file name was provided and exists
  28. load(); // then read the file from disk.
  29. } else { // If the file name was not provided
  30. DataArray = new GBUdbRecord[GBUdbDefaultArraySize]; // then allocate a new Array of
  31. MyArraySize = GBUdbDefaultArraySize; // the default size.
  32. DataArray[ixNextFreeNode()].RawData = // The first new node is the one
  33. GBUdbRootNodeOffset + GBUdbRecordsPerNode; // right after the root node.
  34. DataArray[ixMatchListRoot()].RawData = // Once that's up we can use it to
  35. newMatchNodeRoot(); // allocate the first MatchNode.
  36. }
  37. }
  38. GBUdbDataset::GBUdbDataset(GBUdbDataset& Original) : // Copy constructor.
  39. DataArray(NULL), // The array pointer starts as NULL.
  40. MyArraySize(Original.MyArraySize), // Copy the ArraySize
  41. MyFileName(Original.MyFileName) { // Copy the name pointer.
  42. DataArray = new GBUdbRecord[MyArraySize]; // Allocate a new Array.
  43. memcpy(DataArray, Original.DataArray, sizeof(GBUdbRecord) * MyArraySize); // Copy the data wholesale.
  44. }
  45. const char* GBUdbDataset::FileName(const char* NewName) { // (Re) Set the file name.
  46. MyFileName = ""; // Delete any previous file name.
  47. if(NULL != NewName) { // If we've been given a non-null cstring
  48. MyFileName = NewName; // capture it as our file name.
  49. }
  50. return MyFileName.c_str(); // Return our new FileName.
  51. }
  52. //// During the read, it is safe to plow through the array without
  53. //// checking because any unknown entry points to the zero node and
  54. //// all zero node entries point to the zero node. The read-only
  55. //// method does not add new nodes.
  56. GBUdbRecord& GBUdbDataset::readRecord(unsigned int IP) { // Read a record.
  57. IP = remapIP00toFF(IP); // Make the IP safe for consumption.
  58. int a0, a1, a2, a3; // We will break the IP into 4 octets.
  59. unsigned int xIP = IP; // Grab a copy of IP to maniuplate.
  60. const int LowOctetMask = 0x000000FF; // Mask for seeing the low octet.
  61. const int BitsInOneOctet = 8; // Number of bits to shift per octet.
  62. a3 = xIP & LowOctetMask; xIP >>= BitsInOneOctet; // Grab the a3 octet and shift the IP.
  63. a2 = xIP & LowOctetMask; xIP >>= BitsInOneOctet; // Grab the a2 octet and shift the IP.
  64. a1 = xIP & LowOctetMask; xIP >>= BitsInOneOctet; // Grab the a1 octet and shift the IP.
  65. a0 = xIP & LowOctetMask; // Grab the final octet.
  66. GBUdbIndex RecordIndex = GBUdbRootNodeOffset; // Starting at the root node, follow...
  67. RecordIndex = DataArray[RecordIndex + a0].Index(); // Follow the node then
  68. if(isMatch(RecordIndex)) { // Check for a shortcut (match record).
  69. if(isMatch(RecordIndex, IP)) { return MatchedData(RecordIndex); } // If we have an exact match we're done!
  70. else { return SafeUnknownRecord(); } // If we have a mismatch we are lost...
  71. }
  72. RecordIndex = DataArray[RecordIndex + a1].Index(); // Follow the node then
  73. if(isMatch(RecordIndex)) { // Check for a shortcut (match record).
  74. if(isMatch(RecordIndex, IP)) { return MatchedData(RecordIndex); } // If we have an exact match we're done!
  75. else { return SafeUnknownRecord(); } // If we have a mismatch we are lost...
  76. }
  77. RecordIndex = DataArray[RecordIndex + a2].Index(); // Follow the node. No more match checks.
  78. if(isMatch(RecordIndex)) { // Check for a shortcut (match record).
  79. if(isMatch(RecordIndex, IP)) { return MatchedData(RecordIndex); } // If we have an exact match we're done!
  80. else { return SafeUnknownRecord(); } // If we have a mismatch we are lost...
  81. }
  82. return DataArray[RecordIndex + a3]; // Final node has our data :-)
  83. }
  84. //// dropRecord()
  85. //// This code is essentially a hack of the readRecord() code. If it finds
  86. //// the record it will return true, mark the record as GBUdbUnknown, reduce
  87. //// the IP count, and de-allocate the Match record. Records stored in nodes
  88. //// are set to GBUdbUnknown and the node is left in place - otherwise repeated
  89. //// add and drop operations would lead to leaking all nodes into the match
  90. //// record allocation space. (Node allocation is not a linked list ;-)
  91. bool GBUdbDataset::dropRecord(unsigned int IP) { // Drop an IP record.
  92. IP = remapIP00toFF(IP); // Make the IP safe for consumption.
  93. int a0, a1, a2, a3; // We will break the IP into 4 octets.
  94. unsigned int xIP = IP; // Grab a copy of IP to maniuplate.
  95. const int LowOctetMask = 0x000000FF; // Mask for seeing the low octet.
  96. const int BitsInOneOctet = 8; // Number of bits to shift per octet.
  97. a3 = xIP & LowOctetMask; xIP >>= BitsInOneOctet; // Grab the a3 octet and shift the IP.
  98. a2 = xIP & LowOctetMask; xIP >>= BitsInOneOctet; // Grab the a2 octet and shift the IP.
  99. a1 = xIP & LowOctetMask; xIP >>= BitsInOneOctet; // Grab the a1 octet and shift the IP.
  100. a0 = xIP & LowOctetMask; // Grab the final octet.
  101. GBUdbIndex RecordIndex = GBUdbRootNodeOffset; // Starting at the root node, follow...
  102. GBUdbIndex Node0Index = GBUdbRootNodeOffset; // Keep track of our previous nodes.
  103. GBUdbIndex Node1Index = 0; // This node not set yet.
  104. GBUdbIndex Node2Index = 0; // This node not set yet.
  105. GBUdbIndex Node3Index = 0; // This node not set yet.
  106. RecordIndex = DataArray[Node0Index + a0].Index(); // Follow the node then
  107. if(isMatch(RecordIndex)) { // Check for a shortcut (match record).
  108. if(isMatch(RecordIndex, IP)) { // If we have an exact match we proceed:
  109. MatchedData(RecordIndex).RawData = GBUdbUnknown; // Set the data in the match to unknown.
  110. DataArray[Node0Index + a0].Index(GBUdbUnknown); // Remove the reference to the match record.
  111. deleteMatchAt(RecordIndex); // Reclaim the match record for re-use.
  112. decreaseIPCount(); // Reduce the IP count.
  113. return true; // Return that we were successful.
  114. } else { return false; } // If we have a mismatch we cannot delete.
  115. } else { // If this was a Node link then
  116. Node1Index = RecordIndex; // capture the node root and get ready
  117. } // to follow the next node.
  118. RecordIndex = DataArray[Node1Index + a1].Index(); // Follow the node then
  119. if(isMatch(RecordIndex)) { // Check for a shortcut (match record).
  120. if(isMatch(RecordIndex, IP)) { // If we have an exact match we proceed:
  121. MatchedData(RecordIndex).RawData = GBUdbUnknown; // Set the data in the match to unknown.
  122. DataArray[Node1Index + a1].Index(GBUdbUnknown); // Remove the reference to the match record.
  123. deleteMatchAt(RecordIndex); // Reclaim the match record for re-use.
  124. decreaseIPCount(); // Reduce the IP count.
  125. return true; // Return that we were successful.
  126. } else { return false; } // If we have a mismatch we cannot delete.
  127. } else { // If this was a Node link then
  128. Node2Index = RecordIndex; // capture the node root and get ready
  129. } // to follow the next node.
  130. RecordIndex = DataArray[Node2Index + a2].Index(); // Follow the node then
  131. if(isMatch(RecordIndex)) { // Check for a shortcut (match record).
  132. if(isMatch(RecordIndex, IP)) { // If we have an exact match we proceed:
  133. MatchedData(RecordIndex).RawData = GBUdbUnknown; // Set the data in the match to unknown.
  134. DataArray[Node2Index + a2].Index(GBUdbUnknown); // Remove the reference to the match record.
  135. deleteMatchAt(RecordIndex); // Reclaim the match record for re-use.
  136. decreaseIPCount(); // Reduce the IP count.
  137. return true; // Return that we were successful.
  138. } else { return false; } // If we have a mismatch we cannot delete.
  139. } else { // If this was a Node link then
  140. Node3Index = RecordIndex; // capture the node root and get ready
  141. } // to follow the next node.
  142. RecordIndex = Node3Index + a3; // Follow the node.
  143. if(GBUdbUnknown != DataArray[RecordIndex].RawData) { // If there is data there then
  144. DataArray[RecordIndex].RawData = GBUdbUnknown; // mark the entry as unknown,
  145. decreaseIPCount(); // decrease the IP count
  146. return true; // and return true.
  147. } // If we got all the way to the end and
  148. return false; // didn't find a match then return false.
  149. }
  150. /* Ahhh, the simple life. In a single mode lightning index, each key
  151. ** octet lives in a node, so when you grow a new path you either follow
  152. ** existing nodes or make new ones. We're not doing that here, but as
  153. ** a reference here is how that is usually handled:
  154. **
  155. GBUdbIndex GBUdbDataset::invokeAt(GBUdbRecord& R) { // Invoke at Record.
  156. if(GBUdbUnknown == R.RawData) { // If the record does not point to a
  157. R.Index(newNodeRoot()); // node then give it a new node.
  158. } // If the record already has a node
  159. return R.Index(); // or we gave it one, then follow it.
  160. }
  161. */
  162. //// Little helper function for invokeAt()
  163. int getOctet(int Octet, unsigned int IP) { // Returns Octet number Octet from IP.
  164. const int BitsInOneOctet = 8; // Number of bits to shift per octet.
  165. const int LowOctetMask = 0x000000FF; // Mask for seeing the low octet.
  166. int BitsToShift = 0; // Assume we want a3 but
  167. switch(Octet) { // If we don't, use this handy switch.
  168. case 0: { BitsToShift = 3 * BitsInOneOctet; break; } // For octet 0, shift out 3 octets.
  169. case 1: { BitsToShift = 2 * BitsInOneOctet; break; } // For octet 1, shift out 2 octets.
  170. case 2: { BitsToShift = 1 * BitsInOneOctet; break; } // For octet 2, shift out 1 octets.
  171. } // For octet 3, shift none more octets.
  172. if(0 < BitsToShift) { // If we have bits to shift then
  173. IP >>= BitsToShift; // shift them.
  174. }
  175. return (IP & LowOctetMask); // Exctract the octet at the bottom.
  176. }
  177. //// invokeAt() is a helper function that encapsulates the work of growing new
  178. //// pathways. There are several cases to handle in a bimodal indexing scheme
  179. //// since sometimes you extend new nodes (as commented out above), and some-
  180. //// times you create MatchRecords, and sometimes you have collisions and
  181. //// have to extend previous matches.... or not. All of that will become clear
  182. //// shortly ;-) The good news is that at least invokeAt() is always supposed
  183. //// to return the next place to go --- that is, you never get lost because if
  184. //// the next step in the path does not exist yet then you create it.
  185. GBUdbIndex GBUdbDataset::invokeAt(GBUdbRecord& R, unsigned int IP, int Octet, bool ExtendMatches) {
  186. // R is either known (goes somewhere) or unknown (we would be lost).
  187. // IF R is UNNKOWN then we ...
  188. //// create a match and return it. (No conflict, no extension, no extra node :-)
  189. //**** We got out of that one so we're back at the root level.
  190. if(GBUdbUnknown == R.RawData) {
  191. R.Index(newMatchRecord(IP));
  192. return R.Index();
  193. }
  194. // ELSE R is KNOWN then it either points to a MatchRecord or a Node.
  195. //// IF R points to a Node then we will simply follow it.
  196. //**** We got out of that one so we're back at the root level.
  197. if(!isMatch(R.Index())) {
  198. return R.Index();
  199. }
  200. // ELSE R points to a MatchRecord then we get more complex.
  201. //// IF the MatchRecord matches our IP then we simply follow it.
  202. //**** We got out of that one so we're back at the root level.
  203. if(isMatch(R.Index(),IP)) {
  204. return R.Index();
  205. }
  206. // ELSE the MatchRecord does not match then we get more complex again...
  207. //// IF we are Extending Matches then we...
  208. ////// create a new node
  209. ////// push the existing match onto the new node
  210. ////// and create a new match for the new IP on that node.
  211. ////// since we already have the solution we return the new match node index (skip a step).
  212. //**** We got out of that one so we're back at the root level.
  213. if(ExtendMatches) { // If we are extending matches
  214. GBUdbIndex I = newNodeRoot(); // we create a new node.
  215. int NewSlotForCurrentMatch = // Locate the slot in that node where
  216. getOctet( // the current match should reside
  217. Octet + 1, // based on the octet after this one
  218. DataArray[R.Index()] // by extracting that octet from
  219. .RawData); // the MatchReord header.
  220. // Then we put the current match into
  221. DataArray[I + NewSlotForCurrentMatch].Index(R.Index()); // the correct slot on the new node,
  222. return R.Index(I); // point the current slot to that node
  223. } // and return the node to be followed.
  224. // ELSE we are NOT Extending Matches then we...
  225. // ** KNOW that we are adding node a3 and dealing with the final octet **
  226. //// create a new node
  227. //// map the existing match data into the new node.
  228. //// delete the existing match (for reallocation). deleteMatchAt(GBUdbIndex I)
  229. //// map the new IP into the new node.
  230. GBUdbIndex I = newNodeRoot(); // Create a new node.
  231. int NewSlotForCurrentMatch = // Locate the slot in that node where
  232. getOctet( // the current match should reside
  233. Octet + 1, // based on the octet after this one
  234. DataArray[R.Index()] // by extracting that octet from
  235. .RawData); // the MatchReord header.
  236. if(ExtendMatches) { // If we are extending matches...
  237. // then we put the current match into
  238. DataArray[I + NewSlotForCurrentMatch].Index(R.Index()); // the correct slot on the new node.
  239. } else { // If we are not extending matches...
  240. // then we must be at the end node so
  241. DataArray[I + NewSlotForCurrentMatch].RawData = // we copy in the data from
  242. MatchedData(R.Index()).RawData; // the current MatchRecord,
  243. deleteMatchAt(R.Index()); // and return the MatchRecord for re-use.
  244. }
  245. return R.Index(I); // Point the current slot to new node
  246. } // and return that node index to follow.
  247. //// The "invoke" method creates all of the needed nodes starting
  248. //// at any point where an "unwknown" entry is found.
  249. GBUdbRecord& GBUdbDataset::invokeRecord(unsigned int IP) { // Invoke a record.
  250. if(FreeNodes() < GBUdbGrowthThreshold) grow(); // If we need more space, make more.
  251. IP = remapIP00toFF(IP); // Make the IP safe for consumption.
  252. int a0, a1, a2, a3; // We will break the IP into 4 octets.
  253. unsigned int xIP = IP; // Grab a copy of IP to maniuplate.
  254. const int LowOctetMask = 0x000000FF; // Mask for seeing the low octet.
  255. const bool Extend = true; // Magic number for extending Matches.
  256. const bool DoNotExtend = false; // Magic number for NOT extending them.
  257. const int BitsInOneOctet = 8; // Number of bits to shift per octet.
  258. a3 = xIP & LowOctetMask; xIP >>= BitsInOneOctet; // Grab the a3 octet and shift the IP.
  259. a2 = xIP & LowOctetMask; xIP >>= BitsInOneOctet; // Grab the a2 octet and shift the IP.
  260. a1 = xIP & LowOctetMask; xIP >>= BitsInOneOctet; // Grab the a1 octet and shift the IP.
  261. a0 = xIP & LowOctetMask; // Grab the final octet.
  262. GBUdbIndex RecordIndex = GBUdbRootNodeOffset; // Starting at the root node,
  263. RecordIndex = invokeAt(DataArray[RecordIndex + a0], IP, 0, Extend); // Invoke w/ possible match outcome.
  264. if(isMatch(RecordIndex, IP)) { // If this resulted in a match
  265. GBUdbRecord& Result = MatchedData(RecordIndex); // then we will grab the match data
  266. increaseIPCountIfNew(Result); // and increase the IP count if it's new.
  267. return Result; // Then we return the result. Done!
  268. }
  269. RecordIndex = invokeAt(DataArray[RecordIndex + a1], IP, 1, Extend); // Invode w/ possible match outcome.
  270. if(isMatch(RecordIndex, IP)) { // If this resulted in a match
  271. GBUdbRecord& Result = MatchedData(RecordIndex); // then we will grab the match data
  272. increaseIPCountIfNew(Result); // and increase the IP count if it's new.
  273. return Result; // Then we return the result. Done!
  274. }
  275. RecordIndex = invokeAt(DataArray[RecordIndex + a2], IP, 2, DoNotExtend); // Invode w/ possible match outcome.
  276. if(isMatch(RecordIndex, IP)) { // If this resulted in a match
  277. GBUdbRecord& Result = MatchedData(RecordIndex); // then we will grab the match data
  278. increaseIPCountIfNew(Result); // and increase the IP count if it's new.
  279. return Result; // Then we return the result. Done!
  280. }
  281. GBUdbRecord& Result = DataArray[RecordIndex + a3]; // Grab the record at the final node.
  282. increaseIPCountIfNew(Result); // If new, increase the IP count.
  283. return Result; // Return the record.
  284. }
  285. void GBUdbDataset::save() { // Flush the GBUdb to disk.
  286. string TempFileName = MyFileName + ".tmp"; // Calculate temp and
  287. string BackFileName = MyFileName + ".bak"; // backup file names.
  288. ofstream dbFile; // Grab a file for writing.
  289. dbFile.open(TempFileName.c_str(), ios::out | ios::binary | ios::trunc); // Open the file and truncate if present.
  290. dbFile.write((char*)DataArray, sizeof(GBUdbRecord) * MyArraySize); // Write our array into the file.
  291. bool AllOK = dbFile.good(); // Are we happy with this?
  292. dbFile.close(); // Close the file when done to be nice.
  293. if(AllOK) { // If everything appears to be ok
  294. unlink(BackFileName.c_str()); // Delete any old backup file we have
  295. rename(MyFileName.c_str(), BackFileName.c_str()); // and make the current file a backup.
  296. rename(TempFileName.c_str(), MyFileName.c_str()); // Then make our new file current.
  297. }
  298. }
  299. void GBUdbDataset::load() { // Read the GBUdb from disk.
  300. ifstream dbFile; // Grab a file for reading.
  301. dbFile.open(MyFileName.c_str(), ios::in | ios::binary); // Open the file with the name we have.
  302. dbFile.seekg(0, ios::end); // Go to the end of the
  303. int FileSize = dbFile.tellg(); // file and back so we can
  304. dbFile.seekg(0, ios::beg); // determine it's size.
  305. int SaneGBUdbFileSizeLimit = (GBUdbDefaultArraySize * sizeof(GBUdbRecord)); // What is a sane size limit?
  306. assert(SaneGBUdbFileSizeLimit <= FileSize); // File size sanity check.
  307. int NewArraySize = FileSize / sizeof(GBUdbRecord); // How many records in this file?
  308. if(NULL != DataArray) { // If we have an array loaded then
  309. delete[] DataArray; // delete the array,
  310. DataArray = NULL; // NULL it's pointer,
  311. MyArraySize = 0; // and zero it's size.
  312. }
  313. DataArray = new GBUdbRecord[NewArraySize]; // Allocate an array of the proper size
  314. MyArraySize = NewArraySize; // set the local size variable
  315. dbFile.read((char*)DataArray,FileSize); // and read the file into the array.
  316. dbFile.close(); // Close when done to be nice.
  317. }
  318. void GBUdbDataset::grow(int HowManyNodes) { // Grow the DataArray.
  319. int NewArraySize = MyArraySize + (HowManyNodes * GBUdbRecordsPerNode); // Calcualte the new array size.
  320. GBUdbRecord* NewDataArray = new GBUdbRecord[NewArraySize]; // Allocate the new array.
  321. int OldArrayLessControl = MyArraySize + GBUdbControlNodeOffset; // Include all records but no control.
  322. memcpy(NewDataArray, DataArray, sizeof(GBUdbRecord) * OldArrayLessControl); // Copy the old data to the new array.
  323. for( // Loop through the control nodes...
  324. int o = MyArraySize + GBUdbControlNodeOffset, // o = old node index
  325. n = NewArraySize + GBUdbControlNodeOffset, // n = new node index
  326. c = GBUdbRecordsPerNode; // c = the record count (how many to do).
  327. c > 0; // For until we run out of records,
  328. c--) { // decrementing the count each time,
  329. NewDataArray[n].RawData = DataArray[o].RawData;n++;o++; // Copy the old control data.
  330. }
  331. delete[] DataArray; // Delete the old data array.
  332. DataArray = NewDataArray; // Swap in the new data array.
  333. MyArraySize = NewArraySize; // Correct the size value.
  334. }
  335. GBUdbIndex GBUdbDataset::newMatchRecord(unsigned int IP) { // Allocate a new Match record for IP.
  336. GBUdbIndex I = DataArray[ixMatchListRoot()].RawData; // Grab the root unused Match Record index.
  337. GBUdbRecord& R = DataArray[I]; // Grab the record itself and inspect it.
  338. if((R.RawData & GBUdbFlagsMask) != GBUdbMatchUnusedBit) { // Check that this looks like an
  339. throw MatchAllocationCorrupted(); // unused match record and if not throw!
  340. } // If all is well then lets proceed.
  341. //// First, let's heal the linked list for future allocations.
  342. if(GBUdbMatchUnusedBit == R.RawData) { // If the match record we are on is
  343. DataArray[ixMatchListRoot()].RawData = // the last in the list then allocate
  344. newMatchNodeRoot(); // a new MatchListNode for the next
  345. } else { // allocation. However, if there are
  346. DataArray[ixMatchListRoot()].RawData = // more records left in the list then
  347. (R.RawData & GBUdbMatchDataMask); // set up the next node for the next
  348. } // allocation.
  349. //// Once that's done we can use the record we have for real data.
  350. R.RawData = EncodedMatch(IP); // Encode the match record for the IP.
  351. return I; // Return the match record's index.
  352. }
  353. GBUdbIndex GBUdbDataset::newMatchNodeRoot() { // Allocate a new Match node.
  354. GBUdbIndex I = newNodeRoot(); // Grab a new node to convert.
  355. int iLastMatch = GBUdbRecordsPerNode - 2; // Calc the localized i for last match.
  356. for(int i = 0; i < iLastMatch; i+=2) { // Loop through the node
  357. DataArray[I+i].RawData = GBUdbMatchUnusedBit | (I+i+2); // Build a linked list of Unused Match
  358. DataArray[I+i+1].RawData = GBUdbUnknown; // records with empty data.
  359. }
  360. DataArray[I+iLastMatch].RawData = GBUdbMatchUnusedBit; // The last record gets a NULL index
  361. DataArray[I+iLastMatch+1].RawData = GBUdbUnknown; // and null data to terminate the list.
  362. return I; // Return the root index.
  363. }
  364. // doForAllRecords()
  365. // This method uses a recursive call to doAllAtNode()
  366. // doAllAtNode sweeps through each record in a node and processes any
  367. // node entries through the next level (calling itself) or directly if
  368. // the node is node3, or if it's pointing to a match record.
  369. void GBUdbDataset::updateWorkingIP(unsigned int& WIP, int OctetValue, int Level) { // Update the Working IP (WIP) at octet Level
  370. switch(Level) {
  371. case 0: { // For the node zero address,
  372. WIP = WIP & 0x00FFFFFF; // Mask out the node zero bits.
  373. OctetValue = OctetValue << 24; // Shift the octet value into position.
  374. WIP = WIP | OctetValue; // Or the octet value bits into place.
  375. break;
  376. }
  377. case 1: {
  378. WIP = WIP & 0xFF00FFFF; // Mask out the node zero bits.
  379. OctetValue = OctetValue << 16; // Shift the octet value into position.
  380. WIP = WIP | OctetValue; // Or the octet value bits into place.
  381. break;
  382. }
  383. case 2: {
  384. WIP = WIP & 0xFFFF00FF; // Mask out the node zero bits.
  385. OctetValue = OctetValue << 8; // Shift the octet value into position.
  386. WIP = WIP | OctetValue; // Or the octet value bits into place.
  387. break;
  388. }
  389. case 3: {
  390. WIP = WIP & 0xFFFFFF00; // Mask out the node zero bits.
  391. WIP = WIP | OctetValue; // Or the octet value bits into place.
  392. break;
  393. }
  394. }
  395. }
  396. //// Note about doAllAtNode(). The x.x.x.0 address is skipped on purpose. This
  397. //// is because all x.x.x.0 addresses are mapped to x.x.x.255. By skipping this
  398. //// address and starting at x.x.x.1 in any search, we do not need to check for
  399. //// x.x.x.0 ips that were remapped. They will simply appear at x.x.x.255.
  400. void GBUdbDataset::doAllAtNode( // Recursively call O with all valid records.
  401. GBUdbIndex I, // Input the node index.
  402. GBUdbOperator& O, // Input the Operator to call.
  403. int NodeLevel, // Input the NodeLevel.
  404. unsigned int WIP // Input the working IP.
  405. ) {
  406. int FirstI = (3 > NodeLevel) ? 0 : 1; // Skip any x.x.x.0 addresses.
  407. for(int i = FirstI; i < GBUdbRecordsPerNode; i++) { // Loop through the slots in this node.
  408. GBUdbIndex RecordIndex = DataArray[I + i].Index(); // Get the record index for this slot.
  409. if(GBUdbUnknown != RecordIndex) { // Check that this slot is not empty.
  410. updateWorkingIP(WIP, i, NodeLevel); // If we've got something then update the WIP.
  411. if(3 > NodeLevel) { // If we are working in rootward nodes:
  412. if(isMatch(RecordIndex)) { // Check for a match record. If we have one then
  413. unsigned int MatchIP = WIP & 0xFF000000; // build the IP for the match from the root
  414. MatchIP |= (DataArray[RecordIndex].RawData & 0x00FFFFFF); // of the WIP and the match IP data.
  415. O(MatchIP, MatchedData(RecordIndex)); // Then call the operator with the matched data.
  416. // If this slot is not a match record
  417. } else { // then it is a node address so we will
  418. doAllAtNode(RecordIndex, O, NodeLevel+1, WIP); // recurse to that node at a deeper level.
  419. }
  420. } else { // If we are working in the last node then
  421. O(WIP, DataArray[I + i]); // call the Operator with this IP & Record.
  422. } // All known data values in the last node are
  423. } // actual data records after all.
  424. }
  425. }
  426. void GBUdbDataset::doForAllRecords(GBUdbOperator& O) { // Call O for every valid record.
  427. unsigned int WorkingIP = 0; // A working IP for all levels to use.
  428. int NodeLevel = 0; // The Node level where we start.
  429. doAllAtNode(GBUdbRootNodeOffset, O, NodeLevel, WorkingIP); // Start at the root node, level 0.
  430. }
  431. //// GBUdb Implementations /////////////////////////////////////////////////////
  432. bool AlertFor(int count) { // True if an alert is needed.
  433. return ( // We want an alert whenever a count
  434. 0x00000001 == count || // hits any of these thresholds. Each
  435. 0x00000002 == count || // threshold is a new bit position
  436. 0x00000004 == count || // indicating that the count has
  437. 0x00000008 == count || // achieved a new power of 2. This
  438. 0x00000010 == count || // mechanism insures that newer IPs
  439. 0x00000020 == count || // get lots of attention while long
  440. 0x00000040 == count || // standing IPs still get visited
  441. 0x00000080 == count || // from time to time as their activity
  442. 0x00000100 == count || // continues.
  443. 0x00000200 == count ||
  444. 0x00000400 == count ||
  445. 0x00000800 == count ||
  446. 0x00001000 == count ||
  447. 0x00002000 == count ||
  448. 0x00004000 == count
  449. );
  450. }
  451. char* getTimestamp(char* TimestampBfr) { // Creates an ISO GMT timestamp.
  452. time_t rawtime; // Get a timer and
  453. tm * gmt; // a time structure.
  454. time(&rawtime); // Grab the current time and
  455. gmt=gmtime(&rawtime); // convert it to GMT.
  456. sprintf(TimestampBfr,"%04d%02d%02d%02d%02d%02d", // Format yyyymmddhhmmss
  457. gmt->tm_year+1900,
  458. gmt->tm_mon+1,
  459. gmt->tm_mday,
  460. gmt->tm_hour,
  461. gmt->tm_min,
  462. gmt->tm_sec
  463. );
  464. return TimestampBfr;
  465. }
  466. char* getIPString(unsigned int IP, char* bfr) { // Converts an IP to a string.
  467. int a0, a1, a2, a3; // We will break the IP into 4 octets.
  468. const int LowOctetMask = 0x000000FF; // Mask for seeing the low octet.
  469. const int BitsInOneOctet = 8; // Number of bits to shift per octet.
  470. a3 = IP & LowOctetMask; IP >>= BitsInOneOctet; // Grab the a3 octet and shift the IP.
  471. a2 = IP & LowOctetMask; IP >>= BitsInOneOctet; // Grab the a2 octet and shift the IP.
  472. a1 = IP & LowOctetMask; IP >>= BitsInOneOctet; // Grab the a1 octet and shift the IP.
  473. a0 = IP & LowOctetMask; // Grab the final octet.
  474. sprintf(bfr,"%d.%d.%d.%d",a0,a1,a2,a3);
  475. return bfr;
  476. }
  477. void GBUdb::recordAlertFor(unsigned int IP, GBUdbRecord& R, unsigned int C) { // Record an alert event for R if needed.
  478. if(AlertFor(C)) { // If an alert is needed at this level...
  479. GBUdbAlert NewAlert; // Create a new alert record.
  480. NewAlert.IP = IP; // Assign the IP.
  481. NewAlert.R = R; // Assign the Record.
  482. ScopeMutex JustMe(AlertsMutex); // Lock the alerts list mutex.
  483. MyAlerts.push_back(NewAlert); // Add our new alert to the list.
  484. }
  485. }
  486. GBUdbAlert::GBUdbAlert() : // Default constructor gets timestamp.
  487. IP(0) { // IP to zero, R will init to zero
  488. getTimestamp(UTC); // on it's own... Get timestamp.
  489. }
  490. string GBUdbAlert::toXML() { // Convert this alert to XML text
  491. stringstream Alert; // We'll use a stringstream.
  492. const char* FlagName; // We will want the Flag as text.
  493. switch(R.Flag()) { // Switch on the Flag() value.
  494. case Good: { FlagName = "Good"; break; } // Convert each value to it's name.
  495. case Bad: { FlagName = "Bad"; break; }
  496. case Ugly: { FlagName = "Ugly"; break; }
  497. case Ignore: { FlagName = "Ignore"; break; }
  498. }
  499. char IPStringBfr[20]; // We need a buffer for our IP.
  500. Alert
  501. << "<gbu time=\'" << UTC // GBU alert + timestamp followed
  502. << "\' ip=\'" << getIPString(IP,IPStringBfr) // with the IP,
  503. << "\' t=\'" << FlagName // the type flag,
  504. << "\' b=\'" << R.Bad() // the bad count,
  505. << "\' g=\'" << R.Good() // and the good count.
  506. << "\'/>"; // That's the end.
  507. return Alert.str(); // Return the string.
  508. }
  509. //// Alert import and export - for sharing data between nodes.
  510. void GBUdb::GetAlerts(list<GBUdbAlert>& ListToFill) { // Get all current alerts & clear;
  511. ListToFill.clear(); // Clear out the list to fill.
  512. ScopeMutex JustMe(AlertsMutex); // Lock for a moment.
  513. ListToFill = MyAlerts; // Copy our alerts to the new list.
  514. MyAlerts.clear(); // Clear our alerts.
  515. }
  516. // In order to allow gbudb nodes to interact without swamping their individuality,
  517. // the default mode for integrating thier data is to represent the remote peer's
  518. // influence on a logarithmic scale.
  519. unsigned int rescaleGBUdbCount(unsigned int C) { // Rescale count C for integration.
  520. if(C < 0x00000001) { return 0; } else // Log2, really, .. the short way.
  521. if(C < 0x00000002) { return 1; } else // How many significant bits are in
  522. if(C < 0x00000004) { return 2; } else // the number. Put another way, what
  523. if(C < 0x00000008) { return 3; } else // power of 2 is required to for
  524. if(C < 0x00000010) { return 4; } else // this number.
  525. if(C < 0x00000020) { return 5; } else
  526. if(C < 0x00000040) { return 6; } else
  527. if(C < 0x00000080) { return 7; } else
  528. if(C < 0x00000100) { return 8; } else
  529. if(C < 0x00000200) { return 9; } else
  530. if(C < 0x00000400) { return 10; } else
  531. if(C < 0x00000800) { return 11; } else
  532. if(C < 0x00001000) { return 12; } else
  533. if(C < 0x00002000) { return 13; } else
  534. if(C < 0x00004000) { return 14; } else
  535. return 15;
  536. }
  537. void GBUdb::ImportAlerts(list<GBUdbAlert>& PeerAlerts) { // Integrate peer alerts using log2.
  538. list<GBUdbAlert>::iterator iA;
  539. for(iA = PeerAlerts.begin(); iA != PeerAlerts.end(); iA++) { // Go through the list of PeerAlerts.
  540. GBUdbRecord R = (*iA).R; // Grab the Record in this alert.
  541. R.Bad(rescaleGBUdbCount(R.Bad())); // Adjust the bad and good counts
  542. R.Good(rescaleGBUdbCount(R.Good())); // for integration.
  543. adjustCounts((*iA).IP, R); // Adjust the local counts w/ R.
  544. }
  545. }
  546. //// doForAllRecords
  547. //// This method handles GBUdbOperators and their locking semantics.
  548. //// For full dataset locking the mutex is acquired before calling the
  549. //// dataset's doForAllRecords(). For record locking, the O passed to
  550. //// this method is wrapped in a record locking shim (below) and that is
  551. //// passed to the dataset. If None is selected then the Operator is
  552. //// passed to the dataset as is -- assuming that the Operator will handle
  553. //// it's own locking as needed.
  554. class GBUdbRecordLockingShim : public GBUdbOperator { // Record locking shim for doForAllRecords.
  555. private:
  556. GBUdbOperator& MyOperator; // Reference the Operator we will be servicing.
  557. Mutex& MyMutex; // Reference the Mutex for the GBUdb we are in.
  558. public:
  559. GBUdbRecordLockingShim(GBUdbOperator& O, Mutex M) : // On construction we grab our critical pieces.
  560. MyOperator(O),
  561. MyMutex(M) {
  562. }
  563. GBUdbRecord& operator()(unsigned int IP, GBUdbRecord& R) { // When our operator() is called
  564. ScopeMutex JustMe(MyMutex); // we lock the mutex in scope and
  565. return MyOperator(IP, R); // call the Operator we're servicing.
  566. } // When we leave scope we unlock (see above).
  567. };
  568. void GBUdb::doForAllRecords(GBUdbOperator& O, GBUdbLocking L) { // Calls O(IP, Record) w/Every record.
  569. if(Dataset == L) { // If we are locking for the Dataset, then
  570. ScopeMutex JustMe(MyMutex); // we will lock the mutex during this
  571. MyDataset->doForAllRecords(O); // entire operation.
  572. } else
  573. if(Record == L) { // If we are locking per record then
  574. GBUdbRecordLockingShim X(O, MyMutex); // we create a record locking shim instance
  575. MyDataset->doForAllRecords(X); // and call O() through that.
  576. } else { // If locking is NOT enabled, then
  577. MyDataset->doForAllRecords(O); // we will call O() without any locking.
  578. }
  579. }
  580. //// The saveSnapshot() method allows us to save a snapshot of our dataset
  581. //// while keeping the mutex locked for as short a time as possible: Just long
  582. //// enough to make a copy of the dataset in RAM.
  583. void GBUdb::saveSnapshot() { // Saves a snapshot of the current db.
  584. GBUdbDataset* Snapshot = NULL; // We need a pointer for our snapshot.
  585. if(NULL == MyDataset) { // If we do not have a dataset to copy
  586. return; // then we simply return.
  587. } else { // If we do have a Dataset to copy...
  588. ScopeMutex JustMe(MyMutex); // Lock the mutex and
  589. Snapshot = new GBUdbDataset(*MyDataset); // make a copy in memory.
  590. } // Then we can unlock the mutex.
  591. Snapshot->save(); // Then outside the mutex we can save.
  592. delete Snapshot; // Once saved we can delete the snapshot.
  593. PostsCounter = 0; // Reset the posts counter.
  594. }
  595. //// reduce()
  596. //// Using the doForAllRecords() functionality, this method reduces all counts
  597. //// by 2 thus renormalizing all records at lower count values. Unknown flagged
  598. //// records who's counts drop to zero will achieve the state GBUdbUnknown. As
  599. //// such, those values would not be carried over in a compress() operation.
  600. class ReduceAll : public GBUdbOperator { // To reduce the good and bad counts.
  601. public:
  602. GBUdbRecord& operator()(unsigned int IP, GBUdbRecord& R) { // Given each record,
  603. R.Good(R.Good() >> 1); // Reduce the Good count by half.
  604. R.Bad(R.Bad() >> 1); // Reduce the Bad count by half.
  605. return R; // Return the record.
  606. }
  607. } ReduceAllOperator;
  608. void GBUdb::reduce() { // Reduce all counts by half.
  609. doForAllRecords(ReduceAllOperator); // Call do for all records with the
  610. } // ReduceAllOperator.
  611. //// compress()
  612. //// Using the doForAllRecords() functionality, this method creates a temporary
  613. //// dataset, copies the existing data into that dataset except where the data
  614. //// is GBUdbUnknown, and then swaps the new dataset in place of the old.
  615. class CompressAll : public GBUdbOperator {
  616. private:
  617. GBUdbDataset* MyOldDataset; // Where do we find the old dataset.
  618. GBUdbDataset* MyNewDataset; // Where do we store our new dataset.
  619. int CountConverted;
  620. int CountDropped;
  621. public:
  622. // Note - There is no destructor. It is expected that the calling function
  623. // will extract the NewDataset and replace the OldDataset when the operation
  624. // has been successful.
  625. CompressAll(GBUdbDataset* OldDataset) : // Startup by
  626. MyOldDataset(OldDataset), // Grabbing the old dataset,
  627. MyNewDataset(NULL), // The new one isn't there yet.
  628. CountConverted(0), // Converted and Dropped
  629. CountDropped(0) { // Counts are zero.
  630. MyNewDataset = new GBUdbDataset(NULL); // Allocate a new Dataset.
  631. MyNewDataset->FileName(OldDataset->FileName()); // Set it's name the same as the old.
  632. } // We don't want to Load() it that way ;-)
  633. GBUdbRecord& operator()(unsigned int IP, GBUdbRecord& R) { // The ForAll Operator goes like this...
  634. if(GBUdbUnknown != R.RawData) { // If the record is not GBUdbUnknown then
  635. MyNewDataset->invokeRecord(IP).RawData = R.RawData; // invoke it and copy it's data.
  636. ++CountConverted; // Increment the converted count.
  637. } else { // If the record is GBUdbUnknown then
  638. ++CountDropped; // count it as dropped and forget it.
  639. }
  640. return R; // Return the record reference.
  641. }
  642. GBUdbDataset* Old() {return MyOldDataset;} // Here we can get our OldDataset pointer.
  643. GBUdbDataset* New() {return MyNewDataset;} // Here we can get our NewDataset pointer.
  644. int Converted() {return CountConverted;} // Here we can get the converted count.
  645. int Dropped() {return CountDropped;} // Here we can get the dropped count.
  646. };
  647. void GBUdb::compress() { // Remove any unknown records (reduced to zero).
  648. CompressAll BuildCompressedDataset(MyDataset); // Create a CompressAll operator for this dataset.
  649. ScopeMutex Freeze(MyMutex); // Lock the mutex for the rest of this operation.
  650. MyDataset->doForAllRecords(BuildCompressedDataset); // Copy all of the active data records.
  651. MyDataset = BuildCompressedDataset.New(); // Put the new dataset in place.
  652. delete BuildCompressedDataset.Old(); // Delete the old dataset.
  653. } // All done, so we're unlocked.
  654. int GBUdb::readIgnoreList(const char* FileName) { // setIgnore for a list of IPs
  655. int IPCount = 0; // Keep track of the IPs we read.
  656. try { // Capture any exceptions.
  657. char IPLineBuffer[256]; // Create a line buffer.
  658. ifstream ListFile(FileName, ios::in); // Open up the list file.
  659. while(ListFile.good()) { // While we've got a good file (not eof)
  660. memset(IPLineBuffer, 0, sizeof(IPLineBuffer)); // Clear the buffer.
  661. ListFile.getline(IPLineBuffer, sizeof(IPLineBuffer)); // Read the line.
  662. // Now we have an IP on a line (in theory). We will parse
  663. // the ip and process any that parse correctly.
  664. // First eat anything that's not a digit.
  665. unsigned long IP = 0L; // We need an IP buffer.
  666. char* cursor = IPLineBuffer; // Start on the first byte.
  667. if('#' == *cursor) continue; // Lines that start with # are comments.
  668. // First octet.
  669. while(NULL!=cursor && !isdigit(*cursor)) ++cursor; // Eat any nondigits.
  670. if(!isdigit(*cursor)) continue; // If it's not a digit skip this line.
  671. if(255 < atoi(cursor)) continue; // If the octet is out of range skip!
  672. IP += atoi(cursor); IP <<= 8; // Grab the first int and shift it.
  673. while(isdigit(*cursor)) ++cursor; // Eat those digits.
  674. if('.'!=(*cursor)) continue; // If we don't find a dot skip this line.
  675. ++cursor; // If we do, skip the dot.
  676. // Second octet.
  677. if(!isdigit(*cursor)) continue; // If we're not at digit skip this line.
  678. if(255 < atoi(cursor)) continue; // If the octet is out of range skip!
  679. IP += atoi(cursor); IP <<= 8; // Grab the octet and shift things left.
  680. while(isdigit(*cursor)) ++cursor; // Eat those digits.
  681. if('.'!=(*cursor)) continue; // If we don't find a dot skip this line.
  682. ++cursor; // If we do, skip the dot.
  683. // Third octet.
  684. if(!isdigit(*cursor)) continue; // If we're not at digit skip this line.
  685. if(255 < atoi(cursor)) continue; // If the octet is out of range skip!
  686. IP += atoi(cursor); IP <<= 8; // Grab the octet and shift things left.
  687. while(isdigit(*cursor)) ++cursor; // Eat those digits.
  688. if('.'!=(*cursor)) continue; // If we don't find a dot skip this line.
  689. ++cursor; // If we do, skip the dot.
  690. // Last octet.
  691. if(!isdigit(*cursor)) continue; // If we're not at a digit skip this line.
  692. if(255 < atoi(cursor)) continue; // If the octet is out of range skip!
  693. IP += atoi(cursor); // Grab the octet. IP finished!
  694. setIgnore(IP); // Set the IP to Ignore.
  695. ++IPCount; // Bump the IP count.
  696. }
  697. ListFile.close();
  698. }
  699. catch(...) { } // If we have an exception we stop.
  700. return IPCount; // Always return the number of lines read.
  701. }