// GBUdb.inline.hpp // // (C) Copyright 2006 - 2009 ARM Research Labs, LLC // See www.armresearch.com for the copyright terms. // // See GBUdb.hpp for details & notes. // This file contains inline implementations. //// GBUdbRecord Implementations /////////////////////////////////////////////// inline GBUdbRecord::GBUdbRecord() : // Initialize a new GBUdbRecord RawData(0) { // to ZERO. } inline GBUdbFlag GBUdbRecord::Flag() { // Return the flags. return (GBUdbFlag) (RawData & GBUdbFlagsMask); // Isolate the flags from the data & return. } inline GBUdbFlag GBUdbRecord::Flag(GBUdbFlag f) { // Set the flags. RawData = RawData & (~GBUdbFlagsMask); // Strip the current flags from RawData. RawData = RawData | f; // Put the new flags into RawData. return (GBUdbFlag) (RawData & GBUdbFlagsMask); // Return the flags now in RawData. } inline unsigned int GBUdbRecord::Good() { // Return the Good count. return ((RawData & GBUdbGoodMask) >> GBUdbGoodShift); // Isolate & shift the good count, return. } inline unsigned int GBUdbRecord::Good(unsigned int g) { // Set the good count. RawData = RawData & (~GBUdbGoodMask); // Strip the current good count. g = g & GBUdbLimit; // Make g safe (within bitfield limit). RawData = RawData | (g << GBUdbGoodShift); // Shift & combine g with RawData. return g; // Return the safe g value. } inline unsigned int GBUdbRecord::Bad() { // Get the bad count. return (RawData & GBUdbBadMask); // Isolate the bad data and return. } inline unsigned int GBUdbRecord::Bad(unsigned int b) { // Set the bad count. RawData = RawData & (~GBUdbBadMask); // Strip out the current bad count. b = b & GBUdbLimit; // Make b safe (strip any extra bits). RawData = RawData | b; // Combine RawData with the safe b. return b; // return the safe b. } inline unsigned int GBUdbRecord::addGood(unsigned int g) { // Add to the good count & normalize. unsigned int G = Good(); // Get the good. unsigned int B = Bad(); // Get the bad. G = G + g; // Add the new g to the good. while(G > GBUdbLimit) { // If normalization is required G = G >> 1; // then reduce the new good B = B >> 1; // and bad counts by half } // until things are normalized. Good(G); // Then go ahead and set the Bad(B); // new value(s) into place. return G; // Return the new good count. } inline unsigned int GBUdbRecord::addBad(unsigned int b) { // Add to the bad count & normalize. unsigned int G = Good(); // Get the good. unsigned int B = Bad(); // Get the bad. B = B + b; // Add the new b to the bad. while(B > GBUdbLimit) { // If normalization is required G = G >> 1; // then reduce the new good B = B >> 1; // and bad counts by half } // until things are normalized. Good(G); // Then go ahead and set the Bad(B); // new value(s) into place. return B; // Return the new good count. } inline GBUdbRecord& GBUdbRecord::integrate(GBUdbRecord& A, int LocalWeight, int RemoteWeight) { // Integrate A unsigned int Gl = Good(); // Get the good and unsigned int Bl = Bad(); // bad counts from unsigned int Gr = A.Good(); // the local and unsigned int Br = A.Bad(); // remote records. Gl = (Gl * LocalWeight) + (Gr * RemoteWeight); // Combine the Good and Bl = (Bl * LocalWeight) + (Br * RemoteWeight); // bad counts using the weights. while(Gl > GBUdbLimit || Bl > GBUdbLimit) { // Normalize the counts by Gl = Gl >> 1; // dividing both in half until Bl = Bl >> 1; // they are both within limits. } Good(Gl); // Then set the new Good Bad(Bl); // and bad values and return return *this; // this object. } inline GBUdbIndex GBUdbRecord::Index() { // Read the record as an index. return (GBUdbIndex) RawData; } inline GBUdbIndex GBUdbRecord::Index(GBUdbIndex i) { // Write the index value of the record. RawData = (unsigned int) i; return (GBUdbIndex) RawData; } // Probability is about the ratio of a given event to the total events. // In this case, positive probabilities indicate a tendency toward spam and // negative probabilities indicate a tendency toward ham. inline double GBUdbRecord::Probability() { // Calculate the probability of spam unsigned int G = Good(); // Get the good and unsigned int B = Bad(); // bad counts and double P = 0.0; // grab a double to hold P. if(0 == B + G) { // If we have no counts yet return P; // then return a zero probability. } // If we have counts lets do the math. P = ((double) B - (double) G) / ((double) B + (double) G); // Calculate the differential return P; // probability and return it. } // The confidence we have in a probability is related to the number of samples // that are present. We calculate the confidence on a logarithmic scale between // one sample and half the maximum number by category (good or bad) because // during condensation all counts may be reduced by half. That is, a 100% // confidence is achieved when a record contains a total of half the maximum // number of counts for a single category. inline double GBUdbRecord::Confidence() { // Calculate our confidence in prob. unsigned int Total = Good() + Bad(); // What is our total count of samples. if(0 == Total) return 0.0; // No samples is no confidence. double Confidence = (log((double)Total) / log((double)(GBUdbLimit/2))); // Calculate on a log scale. if(1.0 < Confidence) Confidence = 1.0; // Max confidence is 1.0. return Confidence; // Return the result. } //// GBUdbDataSet Inline Methods /////////////////////////////////////////////// inline GBUdbIndex GBUdbDataset::ixIPCount() { // Index of the IP count for this db. return MyArraySize + GBUdbIPCountOffset; // Return the offest from the end. } inline GBUdbIndex GBUdbDataset::ixNextFreeNode() { // Index of the Next Free Node. return MyArraySize + GBUdbNextFreeNodeOffset; // Return the offset from the end. } inline GBUdbIndex GBUdbDataset::newNodeRoot() { // Allocates a new node, returns offset. if(0 >= FreeNodes()) { // Check that we have free nodes to throw NoFreeNodes(); // allocate. If we don't then throw! } GBUdbIndex NewNode = DataArray[ixNextFreeNode()].Index(); // Grab the next new node index. DataArray[ixNextFreeNode()].Index(NewNode + GBUdbRecordsPerNode); // Move the allocator up a node. return NewNode; // Return the allocated node. } inline int GBUdbDataset::ArraySize() { // Return the current Array Size. return MyArraySize; } inline int GBUdbDataset::FreeNodes() { // Return the number of free nodes. int FreeRecords = MyArraySize - DataArray[ixNextFreeNode()].RawData; // Find the number of records left. int FreeNodes = (FreeRecords / GBUdbRecordsPerNode) - 1; // Convert to nodes and subtract the return FreeNodes; // control node, the return the value. } inline int GBUdbDataset::IPCount() { // Return the IP count. return DataArray[ixIPCount()].RawData; } inline int GBUdbDataset::increaseIPCount() { // When we add an IP to the db. return DataArray[ixIPCount()].RawData++; // Increment and return the IP count. } inline int GBUdbDataset::decreaseIPCount() { // When we drop an IP from the db. return DataArray[ixIPCount()].RawData--; // Decrement and return the IP count. } inline const char* GBUdbDataset::FileName() { // get the file name. return MyFileName.c_str(); } inline unsigned int GBUdbDataset::EncodedMatch(unsigned int IP) { // Encode an IP as a MatchRecord header. return GBUdbMatchEntryBit | (IP & GBUdbMatchDataMask); // Use the MatchEntery bit and as much } // of the remaining IP data as possible. inline bool GBUdbDataset::isMatch(GBUdbIndex I) { // True if record at I is a match record. return (0 != (DataArray[I].RawData & GBUdbMatchEntryBit)); // Get the raw data and check for the bit. } inline bool GBUdbDataset::isMatch(GBUdbIndex I, unsigned int IP) { // True if record at I is a match for IP. return (DataArray[I].RawData == EncodedMatch(IP)); } inline GBUdbRecord& GBUdbDataset::MatchedData(GBUdbIndex I) { // Returns the data for the match at I. return DataArray[I + 1]; // Since I points to the match record we } // return the record immedately after it. inline GBUdbRecord& GBUdbDataset::SafeUnknownRecord() { // Clears and returns the Safe record. MySafeUnknownRecord.RawData = GBUdbUnknown; // Clear the SafeUnknownRecord and return MySafeUnknownRecord; // return it as the result. } inline GBUdbIndex GBUdbDataset::ixMatchListRoot() { // Index of the Match List Root Index. return MyArraySize + GBUdbMatchListOffset; } inline void GBUdbDataset::increaseIPCountIfNew(GBUdbRecord& R) { // If R is GBUdbUnknown, IncreaseIPCount. if(GBUdbUnknown == R.RawData) { increaseIPCount(); } // If new, increase the IP count. } inline unsigned int GBUdbDataset::remapIP00toFF(unsigned int IP) { // Remaps final octet 00 to FF if needed. const int LowOctetMask = 0x000000FF; // Mask for seeing the low octet. if(0 == (IP & LowOctetMask)) { // If the lowest octet is 00 then return (IP | LowOctetMask); // change it to FF and return. } // If the lowest octet is something else return IP; // then return the IP as is. } inline void GBUdbDataset::deleteMatchAt(GBUdbIndex I) { // Recalls MatchRecord at I for reuse. GBUdbIndex Next = DataArray[ixMatchListRoot()].Index(); // Find the current allocation list root. DataArray[I].RawData = (Next | GBUdbMatchUnusedBit); // Point the current match to that root. DataArray[I+1].RawData = GBUdbUnknown; // Clean out any data the match had. DataArray[ixMatchListRoot()].Index(I); // Make this record the list root. } //// GBUdb Implementations ///////////////////////////////////////////////////// inline GBUdb::GBUdb() : // Construct the db as new. PostsCounter(0) { // No posts yet. MyDataset = new GBUdbDataset(NULL); // Construct with no file name. } inline GBUdb::GBUdb(const char* FileName) : // Construct the db from a file. PostsCounter(0) { // No Posts yet. MyDataset = new GBUdbDataset(FileName); // Load the data set by name. } inline GBUdb::~GBUdb() { // Destroy the db object. if(NULL != MyDataset) { // Save first if we can. MyDataset->save(); delete MyDataset; } } inline const char* GBUdb::FileName() { // Return the file name. return MyDataset->FileName(); } inline const char* GBUdb::FileName(const char* NewName) { // Set/Change the file name. return MyDataset->FileName(NewName); } inline void GBUdb::save() { // Save the data. ScopeMutex JustMe(MyMutex); // Lock the mutex during this operation. MyDataset->save(); // Save the dataset. PostsCounter = 0; // Reset the posts counter. } inline void GBUdb::load() { // Load the data. ScopeMutex JustMe(MyMutex); // Lock the mutex during this operation. MyDataset->load(); // Load the dataset. } inline GBUdbRecord GBUdb::addGood(unsigned int IP, int i) { // Count an IP as good. ScopeMutex JustMe(MyMutex); // Lock the mutex during this operation. ++PostsCounter; // Count this as a post. GBUdbRecord& X = MyDataset->invokeRecord(IP); // Invoke the record. unsigned int C = X.addGood(i); // Add a count to the good side. recordAlertFor(IP, X ,C); // Record an alert if required. return X; // Return a copy for analysis. } inline GBUdbRecord GBUdb::addBad(unsigned int IP, int i) { // Count an IP as bad. ScopeMutex JustMe(MyMutex); // Lock the mutex during this operation. ++PostsCounter; // Count this as a post. GBUdbRecord& X = MyDataset->invokeRecord(IP); // Invoke the reocrd. unsigned int C = X.addBad(i); // Add a count to the bad side. recordAlertFor(IP, X, C); // Record an alert if required. return X; // Return a copy for analysis. } inline GBUdbRecord GBUdb::setGood(unsigned int IP) { // Set the flag to Good for this IP. ScopeMutex JustMe(MyMutex); // Lock the mutex during this operation. GBUdbRecord& X = MyDataset->invokeRecord(IP); // Invoke the reocrd. X.Flag(Good); // Set the Good flag. return X; // Return a copy for analysis. } inline GBUdbRecord GBUdb::setBad(unsigned int IP) { // Set the flag to Bad for this IP. ScopeMutex JustMe(MyMutex); // Lock the mutex during this operation. GBUdbRecord& X = MyDataset->invokeRecord(IP); // Invoke the reocrd. X.Flag(Bad); // Set the Bad flag. return X; // Return a copy for analysis. } inline GBUdbRecord GBUdb::setUgly(unsigned int IP) { // Set the flag to Ugly for this IP. ScopeMutex JustMe(MyMutex); // Lock the mutex during this operation. GBUdbRecord& X = MyDataset->invokeRecord(IP); // Invoke the reocrd. X.Flag(Ugly); // Set the Ugly flag. return X; // Return a copy for analysis. } inline GBUdbRecord GBUdb::setIgnore(unsigned int IP) { // Set the flag to Ignore for this IP. ScopeMutex JustMe(MyMutex); // Lock the mutex during this operation. GBUdbRecord& X = MyDataset->invokeRecord(IP); // Invoke the reocrd. X.Flag(Ignore); // Set the Ignore flag. return X; // Return a copy for analysis. } inline GBUdbRecord GBUdb::getRecord(unsigned int IP) { // Retrieve an IP record. ScopeMutex JustMe(MyMutex); // Lock the mutex during this operation. GBUdbRecord& X = MyDataset->readRecord(IP); // ReadOnly the reocrd. return X; // Return a copy for analysis. } inline GBUdbRecord GBUdb::setRecord(unsigned int IP, GBUdbRecord& R) { // Store an IP record. ScopeMutex JustMe(MyMutex); // Lock the mutex during this operation. GBUdbRecord& X = MyDataset->invokeRecord(IP); // Invoke the reocrd. X = R; // Overwrite X with R. return X; // Return a copy for analysis. } inline GBUdbRecord GBUdb::adjustCounts(unsigned int IP, GBUdbRecord& R) { // Adds counts from R to record for IP. ScopeMutex JustMe(MyMutex); // Lock the data for this operation. GBUdbRecord& X = MyDataset->invokeRecord(IP); // Locate the record in the data. X.Bad(X.Bad() + R.Bad()); // Add the reflected adjustments X.Good(X.Good() + R.Good()); // to the good and bad counts. return X; // Return a copy for analysis. } inline bool GBUdb::dropRecord(unsigned int IP) { // Drop an IP record. ScopeMutex JustMe(MyMutex); // Lock the mutex during this operation. return MyDataset->dropRecord(IP); // Pass on this call to our dataset. } inline int GBUdb::IPCount() { // Number of IPs stored. ScopeMutex JustMe(MyMutex); return MyDataset->IPCount(); } inline int GBUdb::Size() { // Size of GBUdb in bytes. ScopeMutex JustMe(MyMutex); // Lock the mutex during this operation. return MyDataset->ArraySize() * sizeof(GBUdbRecord); // Total records converted to bytes. } inline double GBUdb::Utilization() { // Utilization (percent). ScopeMutex JustMe(MyMutex); // Lock the mutex during this operation. int TotalRecords = MyDataset->ArraySize(); // Calculate the total number of records. int FreeRecords = MyDataset->FreeNodes() * GBUdbRecordsPerNode; // Calculate the number of unused records. int UsedRecords = TotalRecords - FreeRecords; // Calcualte the number of used records. return // Calculate and return as double... ((double) UsedRecords) * 100.0 / // (Used Records * 100) / (TotalRecords) ((double) TotalRecords); } inline int GBUdb::Posts() { // Number of posts since last snapshot. int CurrentCount = PostsCounter; // Grab the current posts count. return CurrentCount; // Return the count we had. }