madscientist
/
SNFMulti
Watch 1
Star 0
Fork 0
Code Issues 0 Pull Requests 0 Releases 0 Wiki Activity
Browse Source

cleaned up namespace and dropped inline GBUdb

master
Pete McNeil 4 years ago
parent
cb4666686f
commit
9eabda2dc3
3 changed files with 382 additions and 394 deletions
Unified View Show Diff Stats
  1. 372
    23
      GBUdb.cpp
  2. 10
    17
      GBUdb.hpp
  3. 0
    354
      GBUdb.inline.hpp

+ 372
- 23
GBUdb.cpp View File

// GBUdb.cpp // GBUdb.cpp
// //
// (C) Copyright 2006 - 2009 ARM Research Labs, LLC
// (C) Copyright 2006 - 2020 ARM Research Labs, LLC
// See www.armresearch.com for the copyright terms. // See www.armresearch.com for the copyright terms.
// //
// See GBUdb.hpp for details. // See GBUdb.hpp for details.


//// Handy utilities... //// Handy utilities...


//// GBUdbRecord Implementations ///////////////////////////////////////////////

GBUdbRecord::GBUdbRecord() : // Initialize a new GBUdbRecord
RawData(0) { // to ZERO.
}

GBUdbFlag GBUdbRecord::Flag() { // Return the flags.
return (GBUdbFlag) (RawData & GBUdbFlagsMask); // Isolate the flags from the data & return.
}

GBUdbFlag GBUdbRecord::Flag(GBUdbFlag f) { // Set the flags.
RawData = RawData & (~GBUdbFlagsMask); // Strip the current flags from RawData.
RawData = RawData | f; // Put the new flags into RawData.
return (GBUdbFlag) (RawData & GBUdbFlagsMask); // Return the flags now in RawData.
}

unsigned int GBUdbRecord::Good() { // Return the Good count.
return ((RawData & GBUdbGoodMask) >> GBUdbGoodShift); // Isolate & shift the good count, return.
}

unsigned int GBUdbRecord::Good(unsigned int g) { // Set the good count.
RawData = RawData & (~GBUdbGoodMask); // Strip the current good count.
g = g & GBUdbLimit; // Make g safe (within bitfield limit).
RawData = RawData | (g << GBUdbGoodShift); // Shift & combine g with RawData.
return g; // Return the safe g value.
}

unsigned int GBUdbRecord::Bad() { // Get the bad count.
return (RawData & GBUdbBadMask); // Isolate the bad data and return.
}

unsigned int GBUdbRecord::Bad(unsigned int b) { // Set the bad count.
RawData = RawData & (~GBUdbBadMask); // Strip out the current bad count.
b = b & GBUdbLimit; // Make b safe (strip any extra bits).
RawData = RawData | b; // Combine RawData with the safe b.
return b; // return the safe b.
}

unsigned int GBUdbRecord::addGood(unsigned int g) { // Add to the good count & normalize.
unsigned int G = Good(); // Get the good.
unsigned int B = Bad(); // Get the bad.
G = G + g; // Add the new g to the good.
while(G > GBUdbLimit) { // If normalization is required
G = G >> 1; // then reduce the new good
B = B >> 1; // and bad counts by half
} // until things are normalized.
Good(G); // Then go ahead and set the
Bad(B); // new value(s) into place.
return G; // Return the new good count.
}

unsigned int GBUdbRecord::addBad(unsigned int b) { // Add to the bad count & normalize.
unsigned int G = Good(); // Get the good.
unsigned int B = Bad(); // Get the bad.
B = B + b; // Add the new b to the bad.
while(B > GBUdbLimit) { // If normalization is required
G = G >> 1; // then reduce the new good
B = B >> 1; // and bad counts by half
} // until things are normalized.
Good(G); // Then go ahead and set the
Bad(B); // new value(s) into place.
return B; // Return the new good count.
}

GBUdbRecord& GBUdbRecord::integrate(GBUdbRecord& A, int LocalWeight, int RemoteWeight) { // Integrate A

unsigned int Gl = Good(); // Get the good and
unsigned int Bl = Bad(); // bad counts from
unsigned int Gr = A.Good(); // the local and
unsigned int Br = A.Bad(); // remote records.

Gl = (Gl * LocalWeight) + (Gr * RemoteWeight); // Combine the Good and
Bl = (Bl * LocalWeight) + (Br * RemoteWeight); // bad counts using the weights.

while(Gl > GBUdbLimit || Bl > GBUdbLimit) { // Normalize the counts by
Gl = Gl >> 1; // dividing both in half until
Bl = Bl >> 1; // they are both within limits.
}
Good(Gl); // Then set the new Good
Bad(Bl); // and bad values and return
return *this; // this object.
}

GBUdbIndex GBUdbRecord::Index() { // Read the record as an index.
return (GBUdbIndex) RawData;
}

GBUdbIndex GBUdbRecord::Index(GBUdbIndex i) { // Write the index value of the record.
RawData = (unsigned int) i;
return (GBUdbIndex) RawData;
}

// Probability is about the ratio of a given event to the total events.
// In this case, positive probabilities indicate a tendency toward spam and
// negative probabilities indicate a tendency toward ham.

double GBUdbRecord::Probability() { // Calculate the probability of spam
unsigned int G = Good(); // Get the good and
unsigned int B = Bad(); // bad counts and
double P = 0.0; // grab a double to hold P.
if(0 == B + G) { // If we have no counts yet
return P; // then return a zero probability.
} // If we have counts lets do the math.
P = ((double) B - (double) G) / ((double) B + (double) G); // Calculate the differential
return P; // probability and return it.
}

// The confidence we have in a probability is related to the number of samples
// that are present. We calculate the confidence on a logarithmic scale between
// one sample and half the maximum number by category (good or bad) because
// during condensation all counts may be reduced by half. That is, a 100%
// confidence is achieved when a record contains a total of half the maximum
// number of counts for a single category.

double GBUdbRecord::Confidence() { // Calculate our confidence in prob.
unsigned int Total = Good() + Bad(); // What is our total count of samples.
if(0 == Total) return 0.0; // No samples is no confidence.
double Confidence = (log((double)Total) / log((double)(GBUdbLimit/2))); // Calculate on a log scale.
if(1.0 < Confidence) Confidence = 1.0; // Max confidence is 1.0.
return Confidence; // Return the result.
}

//// GBUdbDataSet Inline Methods ///////////////////////////////////////////////

GBUdbIndex GBUdbDataset::ixIPCount() { // Index of the IP count for this db.
return MyArraySize + GBUdbIPCountOffset; // Return the offest from the end.
}

GBUdbIndex GBUdbDataset::ixNextFreeNode() { // Index of the Next Free Node.
return MyArraySize + GBUdbNextFreeNodeOffset; // Return the offset from the end.
}

GBUdbIndex GBUdbDataset::newNodeRoot() { // Allocates a new node, returns offset.
if(0 >= FreeNodes()) { // Check that we have free nodes to
throw NoFreeNodes(); // allocate. If we don't then throw!
}
GBUdbIndex NewNode = DataArray[ixNextFreeNode()].Index(); // Grab the next new node index.
DataArray[ixNextFreeNode()].Index(NewNode + GBUdbRecordsPerNode); // Move the allocator up a node.
return NewNode; // Return the allocated node.
}

int GBUdbDataset::ArraySize() { // Return the current Array Size.
return MyArraySize;
}

int GBUdbDataset::FreeNodes() { // Return the number of free nodes.
int FreeRecords = MyArraySize - DataArray[ixNextFreeNode()].RawData; // Find the number of records left.
int FreeNodes = (FreeRecords / GBUdbRecordsPerNode) - 1; // Convert to nodes and subtract the
return FreeNodes; // control node, the return the value.
}

int GBUdbDataset::IPCount() { // Return the IP count.
return DataArray[ixIPCount()].RawData;
}

int GBUdbDataset::increaseIPCount() { // When we add an IP to the db.
return DataArray[ixIPCount()].RawData++; // Increment and return the IP count.
}

int GBUdbDataset::decreaseIPCount() { // When we drop an IP from the db.
return DataArray[ixIPCount()].RawData--; // Decrement and return the IP count.
}

const char* GBUdbDataset::FileName() { // get the file name.
return MyFileName.c_str();
}

unsigned int GBUdbDataset::EncodedMatch(unsigned int IP) { // Encode an IP as a MatchRecord header.
return GBUdbMatchEntryBit | (IP & GBUdbMatchDataMask); // Use the MatchEntery bit and as much
} // of the remaining IP data as possible.

bool GBUdbDataset::isMatch(GBUdbIndex I) { // True if record at I is a match record.
return (0 != (DataArray[I].RawData & GBUdbMatchEntryBit)); // Get the raw data and check for the bit.
}

bool GBUdbDataset::isMatch(GBUdbIndex I, unsigned int IP) { // True if record at I is a match for IP.
return (DataArray[I].RawData == EncodedMatch(IP));
}

GBUdbRecord& GBUdbDataset::MatchedData(GBUdbIndex I) { // Returns the data for the match at I.
return DataArray[I + 1]; // Since I points to the match record we
} // return the record immedately after it.

GBUdbRecord& GBUdbDataset::SafeUnknownRecord() { // Clears and returns the Safe record.
MySafeUnknownRecord.RawData = GBUdbUnknown; // Clear the SafeUnknownRecord and
return MySafeUnknownRecord; // return it as the result.
}

GBUdbIndex GBUdbDataset::ixMatchListRoot() { // Index of the Match List Root Index.
return MyArraySize + GBUdbMatchListOffset;
}

void GBUdbDataset::increaseIPCountIfNew(GBUdbRecord& R) { // If R is GBUdbUnknown, IncreaseIPCount.
if(GBUdbUnknown == R.RawData) { increaseIPCount(); } // If new, increase the IP count.
}

unsigned int GBUdbDataset::remapIP00toFF(unsigned int IP) { // Remaps final octet 00 to FF if needed.
const int LowOctetMask = 0x000000FF; // Mask for seeing the low octet.
if(0 == (IP & LowOctetMask)) { // If the lowest octet is 00 then
return (IP | LowOctetMask); // change it to FF and return.
} // If the lowest octet is something else
return IP; // then return the IP as is.
}

void GBUdbDataset::deleteMatchAt(GBUdbIndex I) { // Recalls MatchRecord at I for reuse.
GBUdbIndex Next = DataArray[ixMatchListRoot()].Index(); // Find the current allocation list root.
DataArray[I].RawData = (Next | GBUdbMatchUnusedBit); // Point the current match to that root.
DataArray[I+1].RawData = GBUdbUnknown; // Clean out any data the match had.
DataArray[ixMatchListRoot()].Index(I); // Make this record the list root.
}

//// GBUdb Implementations /////////////////////////////////////////////////////

GBUdb::GBUdb() : // Construct the db as new.
PostsCounter(0) { // No posts yet.
MyDataset = new GBUdbDataset(NULL); // Construct with no file name.
}

GBUdb::GBUdb(const char* FileName) : // Construct the db from a file.
PostsCounter(0) { // No Posts yet.
MyDataset = new GBUdbDataset(FileName); // Load the data set by name.
}

GBUdb::~GBUdb() { // Destroy the db object.
if(NULL != MyDataset) { // Save first if we can.
MyDataset->save();
delete MyDataset;
}
}

const char* GBUdb::FileName() { // Return the file name.
return MyDataset->FileName();
}

const char* GBUdb::FileName(const char* NewName) { // Set/Change the file name.
return MyDataset->FileName(NewName);
}

void GBUdb::save() { // Save the data.
cd::ScopeMutex JustMe(MyMutex); // Lock the mutex during this operation.
MyDataset->save(); // Save the dataset.
PostsCounter = 0; // Reset the posts counter.
}

void GBUdb::load() { // Load the data.
cd::ScopeMutex JustMe(MyMutex); // Lock the mutex during this operation.
MyDataset->load(); // Load the dataset.
}

GBUdbRecord GBUdb::addGood(unsigned int IP, int i) { // Count an IP as good.
cd::ScopeMutex JustMe(MyMutex); // Lock the mutex during this operation.
++PostsCounter; // Count this as a post.
GBUdbRecord& X = MyDataset->invokeRecord(IP); // Invoke the record.
unsigned int C = X.addGood(i); // Add a count to the good side.
recordAlertFor(IP, X ,C); // Record an alert if required.
return X; // Return a copy for analysis.
}

GBUdbRecord GBUdb::addBad(unsigned int IP, int i) { // Count an IP as bad.
cd::ScopeMutex JustMe(MyMutex); // Lock the mutex during this operation.
++PostsCounter; // Count this as a post.
GBUdbRecord& X = MyDataset->invokeRecord(IP); // Invoke the reocrd.
unsigned int C = X.addBad(i); // Add a count to the bad side.
recordAlertFor(IP, X, C); // Record an alert if required.
return X; // Return a copy for analysis.
}

GBUdbRecord GBUdb::setGood(unsigned int IP) { // Set the flag to Good for this IP.
cd::ScopeMutex JustMe(MyMutex); // Lock the mutex during this operation.
GBUdbRecord& X = MyDataset->invokeRecord(IP); // Invoke the reocrd.
X.Flag(Good); // Set the Good flag.
return X; // Return a copy for analysis.
}

GBUdbRecord GBUdb::setBad(unsigned int IP) { // Set the flag to Bad for this IP.
cd::ScopeMutex JustMe(MyMutex); // Lock the mutex during this operation.
GBUdbRecord& X = MyDataset->invokeRecord(IP); // Invoke the reocrd.
X.Flag(Bad); // Set the Bad flag.
return X; // Return a copy for analysis.
}

GBUdbRecord GBUdb::setUgly(unsigned int IP) { // Set the flag to Ugly for this IP.
cd::ScopeMutex JustMe(MyMutex); // Lock the mutex during this operation.
GBUdbRecord& X = MyDataset->invokeRecord(IP); // Invoke the reocrd.
X.Flag(Ugly); // Set the Ugly flag.
return X; // Return a copy for analysis.
}

GBUdbRecord GBUdb::setIgnore(unsigned int IP) { // Set the flag to Ignore for this IP.
cd::ScopeMutex JustMe(MyMutex); // Lock the mutex during this operation.
GBUdbRecord& X = MyDataset->invokeRecord(IP); // Invoke the reocrd.
X.Flag(Ignore); // Set the Ignore flag.
return X; // Return a copy for analysis.
}


GBUdbRecord GBUdb::getRecord(unsigned int IP) { // Retrieve an IP record.
cd::ScopeMutex JustMe(MyMutex); // Lock the mutex during this operation.
GBUdbRecord& X = MyDataset->readRecord(IP); // ReadOnly the reocrd.
return X; // Return a copy for analysis.
}

GBUdbRecord GBUdb::setRecord(unsigned int IP, GBUdbRecord& R) { // Store an IP record.
cd::ScopeMutex JustMe(MyMutex); // Lock the mutex during this operation.
GBUdbRecord& X = MyDataset->invokeRecord(IP); // Invoke the reocrd.
X = R; // Overwrite X with R.
return X; // Return a copy for analysis.
}

GBUdbRecord GBUdb::adjustCounts(unsigned int IP, GBUdbRecord& R) { // Adds counts from R to record for IP.
cd::ScopeMutex JustMe(MyMutex); // Lock the data for this operation.
GBUdbRecord& X = MyDataset->invokeRecord(IP); // Locate the record in the data.
X.Bad(X.Bad() + R.Bad()); // Add the reflected adjustments
X.Good(X.Good() + R.Good()); // to the good and bad counts.
return X; // Return a copy for analysis.
}

bool GBUdb::dropRecord(unsigned int IP) { // Drop an IP record.
cd::ScopeMutex JustMe(MyMutex); // Lock the mutex during this operation.
return MyDataset->dropRecord(IP); // Pass on this call to our dataset.
}

int GBUdb::IPCount() { // Number of IPs stored.
cd::ScopeMutex JustMe(MyMutex);
return MyDataset->IPCount();
}

int GBUdb::Size() { // Size of GBUdb in bytes.
cd::ScopeMutex JustMe(MyMutex); // Lock the mutex during this operation.
return MyDataset->ArraySize() * sizeof(GBUdbRecord); // Total records converted to bytes.
}

double GBUdb::Utilization() { // Utilization (percent).
cd::ScopeMutex JustMe(MyMutex); // Lock the mutex during this operation.
int TotalRecords = MyDataset->ArraySize(); // Calculate the total number of records.
int FreeRecords = MyDataset->FreeNodes() * GBUdbRecordsPerNode; // Calculate the number of unused records.
int UsedRecords = TotalRecords - FreeRecords; // Calcualte the number of used records.
return // Calculate and return as double...
((double) UsedRecords) * 100.0 / // (Used Records * 100) / (TotalRecords)
((double) TotalRecords);
}

int GBUdb::Posts() { // Number of posts since last snapshot.
int CurrentCount = PostsCounter; // Grab the current posts count.
return CurrentCount; // Return the count we had.
}

//// GBUdbDataset implementations ////////////////////////////////////////////// //// GBUdbDataset implementations //////////////////////////////////////////////


GBUdbDataset::~GBUdbDataset() { // Shutdown a dataset. GBUdbDataset::~GBUdbDataset() { // Shutdown a dataset.
} }


void GBUdbDataset::save() { // Flush the GBUdb to disk. void GBUdbDataset::save() { // Flush the GBUdb to disk.
string TempFileName = MyFileName + ".tmp"; // Calculate temp and
string BackFileName = MyFileName + ".bak"; // backup file names.
ofstream dbFile; // Grab a file for writing.
dbFile.open(TempFileName.c_str(), ios::out | ios::binary | ios::trunc); // Open the file and truncate if present.
std::string TempFileName = MyFileName + ".tmp"; // Calculate temp and
std::string BackFileName = MyFileName + ".bak"; // backup file names.
std::ofstream dbFile; // Grab a file for writing.
dbFile.open(TempFileName.c_str(), // Open the file in binary mode
std::ios::out | std::ios::binary | std::ios::trunc); // and truncate if present.

dbFile.write((char*)DataArray, sizeof(GBUdbRecord) * MyArraySize); // Write our array into the file. dbFile.write((char*)DataArray, sizeof(GBUdbRecord) * MyArraySize); // Write our array into the file.
bool AllOK = dbFile.good(); // Are we happy with this? bool AllOK = dbFile.good(); // Are we happy with this?
dbFile.close(); // Close the file when done to be nice. dbFile.close(); // Close the file when done to be nice.


void GBUdbDataset::load() { // Read the GBUdb from disk. void GBUdbDataset::load() { // Read the GBUdb from disk.


ifstream dbFile; // Grab a file for reading.
dbFile.open(MyFileName.c_str(), ios::in | ios::binary); // Open the file with the name we have.
dbFile.seekg(0, ios::end); // Go to the end of the
std::ifstream dbFile; // Grab a file for reading.
dbFile.open(MyFileName.c_str(), std::ios::in | std::ios::binary); // Open the file with the name we have.
dbFile.seekg(0, std::ios::end); // Go to the end of the
int FileSize = dbFile.tellg(); // file and back so we can int FileSize = dbFile.tellg(); // file and back so we can
dbFile.seekg(0, ios::beg); // determine it's size.
dbFile.seekg(0, std::ios::beg); // determine it's size.


int SaneGBUdbFileSizeLimit = (GBUdbDefaultArraySize * sizeof(GBUdbRecord)); // What is a sane size limit? int SaneGBUdbFileSizeLimit = (GBUdbDefaultArraySize * sizeof(GBUdbRecord)); // What is a sane size limit?
SaneFileSizeCheck(SaneGBUdbFileSizeLimit <= FileSize); // File size sanity check. SaneFileSizeCheck(SaneGBUdbFileSizeLimit <= FileSize); // File size sanity check.
GBUdbAlert NewAlert; // Create a new alert record. GBUdbAlert NewAlert; // Create a new alert record.
NewAlert.IP = IP; // Assign the IP. NewAlert.IP = IP; // Assign the IP.
NewAlert.R = R; // Assign the Record. NewAlert.R = R; // Assign the Record.
ScopeMutex JustMe(AlertsMutex); // Lock the alerts list mutex.
cd::ScopeMutex JustMe(AlertsMutex); // Lock the alerts list mutex.
MyAlerts.push_back(NewAlert); // Add our new alert to the list. MyAlerts.push_back(NewAlert); // Add our new alert to the list.
} }
} }
getTimestamp(UTC); // on it's own... Get timestamp. getTimestamp(UTC); // on it's own... Get timestamp.
} }


string GBUdbAlert::toXML() { // Convert this alert to XML text
stringstream Alert; // We'll use a stringstream.
std::string GBUdbAlert::toXML() { // Convert this alert to XML text
std::stringstream Alert; // We'll use a stringstream.


const char* FlagName = "ERROR"; // We will want the Flag as text. const char* FlagName = "ERROR"; // We will want the Flag as text.
switch(R.Flag()) { // Switch on the Flag() value. switch(R.Flag()) { // Switch on the Flag() value.


//// Alert import and export - for sharing data between nodes. //// Alert import and export - for sharing data between nodes.


void GBUdb::GetAlerts(list<GBUdbAlert>& ListToFill) { // Get all current alerts & clear;
void GBUdb::GetAlerts(std::list<GBUdbAlert>& ListToFill) { // Get all current alerts & clear;
ListToFill.clear(); // Clear out the list to fill. ListToFill.clear(); // Clear out the list to fill.
ScopeMutex JustMe(AlertsMutex); // Lock for a moment.
cd::ScopeMutex JustMe(AlertsMutex); // Lock for a moment.
ListToFill = MyAlerts; // Copy our alerts to the new list. ListToFill = MyAlerts; // Copy our alerts to the new list.
MyAlerts.clear(); // Clear our alerts. MyAlerts.clear(); // Clear our alerts.
} }
return 15; return 15;
} }


void GBUdb::ImportAlerts(list<GBUdbAlert>& PeerAlerts) { // Integrate peer alerts using log2.
list<GBUdbAlert>::iterator iA;
void GBUdb::ImportAlerts(std::list<GBUdbAlert>& PeerAlerts) { // Integrate peer alerts using log2.
std::list<GBUdbAlert>::iterator iA;
for(iA = PeerAlerts.begin(); iA != PeerAlerts.end(); iA++) { // Go through the list of PeerAlerts. for(iA = PeerAlerts.begin(); iA != PeerAlerts.end(); iA++) { // Go through the list of PeerAlerts.
GBUdbRecord R = (*iA).R; // Grab the Record in this alert. GBUdbRecord R = (*iA).R; // Grab the Record in this alert.
R.Bad(rescaleGBUdbCount(R.Bad())); // Adjust the bad and good counts R.Bad(rescaleGBUdbCount(R.Bad())); // Adjust the bad and good counts
private: private:


GBUdbOperator& MyOperator; // Reference the Operator we will be servicing. GBUdbOperator& MyOperator; // Reference the Operator we will be servicing.
Mutex& MyMutex; // Reference the Mutex for the GBUdb we are in.
cd::Mutex& MyMutex; // Reference the Mutex for the GBUdb we are in.


public: public:


GBUdbRecordLockingShim(GBUdbOperator& O, Mutex& M) : // On construction we grab our critical pieces.
GBUdbRecordLockingShim(GBUdbOperator& O, cd::Mutex& M) : // On construction we grab our critical pieces.
MyOperator(O), MyOperator(O),
MyMutex(M) { MyMutex(M) {
} }


GBUdbRecord& operator()(unsigned int IP, GBUdbRecord& R) { // When our operator() is called GBUdbRecord& operator()(unsigned int IP, GBUdbRecord& R) { // When our operator() is called
ScopeMutex JustMe(MyMutex); // we lock the mutex in scope and
cd::ScopeMutex JustMe(MyMutex); // we lock the mutex in scope and
return MyOperator(IP, R); // call the Operator we're servicing. return MyOperator(IP, R); // call the Operator we're servicing.
} // When we leave scope we unlock (see above). } // When we leave scope we unlock (see above).
}; };


void GBUdb::doForAllRecords(GBUdbOperator& O, GBUdbLocking L) { // Calls O(IP, Record) w/Every record. void GBUdb::doForAllRecords(GBUdbOperator& O, GBUdbLocking L) { // Calls O(IP, Record) w/Every record.
if(Dataset == L) { // If we are locking for the Dataset, then if(Dataset == L) { // If we are locking for the Dataset, then
ScopeMutex JustMe(MyMutex); // we will lock the mutex during this
cd::ScopeMutex JustMe(MyMutex); // we will lock the mutex during this
MyDataset->doForAllRecords(O); // entire operation. MyDataset->doForAllRecords(O); // entire operation.
} else } else
if(Record == L) { // If we are locking per record then if(Record == L) { // If we are locking per record then
if(NULL == MyDataset) { // If we do not have a dataset to copy if(NULL == MyDataset) { // If we do not have a dataset to copy
return; // then we simply return. return; // then we simply return.
} else { // If we do have a Dataset to copy... } else { // If we do have a Dataset to copy...
ScopeMutex JustMe(MyMutex); // Lock the mutex and
cd::ScopeMutex JustMe(MyMutex); // Lock the mutex and
Snapshot = new GBUdbDataset(*MyDataset); // make a copy in memory. Snapshot = new GBUdbDataset(*MyDataset); // make a copy in memory.
} // Then we can unlock the mutex. } // Then we can unlock the mutex.
Snapshot->save(); // Then outside the mutex we can save. Snapshot->save(); // Then outside the mutex we can save.


void GBUdb::compress() { // Remove any unknown records (reduced to zero). void GBUdb::compress() { // Remove any unknown records (reduced to zero).
CompressAll BuildCompressedDataset(MyDataset); // Create a CompressAll operator for this dataset. CompressAll BuildCompressedDataset(MyDataset); // Create a CompressAll operator for this dataset.
ScopeMutex Freeze(MyMutex); // Lock the mutex for the rest of this operation.
cd::ScopeMutex Freeze(MyMutex); // Lock the mutex for the rest of this operation.
MyDataset->doForAllRecords(BuildCompressedDataset); // Copy all of the active data records. MyDataset->doForAllRecords(BuildCompressedDataset); // Copy all of the active data records.
MyDataset = BuildCompressedDataset.New(); // Put the new dataset in place. MyDataset = BuildCompressedDataset.New(); // Put the new dataset in place.
delete BuildCompressedDataset.Old(); // Delete the old dataset. delete BuildCompressedDataset.Old(); // Delete the old dataset.
try { // Capture any exceptions. try { // Capture any exceptions.
char IPLineBuffer[256]; // Create a line buffer. char IPLineBuffer[256]; // Create a line buffer.
const int SafeBufferSize = sizeof(IPLineBuffer) - 1; // Safe size always leaves a NULL on the end. const int SafeBufferSize = sizeof(IPLineBuffer) - 1; // Safe size always leaves a NULL on the end.
ifstream ListFile(FileName, ios::in); // Open up the list file.
std::ifstream ListFile(FileName, std::ios::in); // Open up the list file.
while(ListFile.good()) { // While we've got a good file (not eof) while(ListFile.good()) { // While we've got a good file (not eof)
memset(IPLineBuffer, 0, sizeof(IPLineBuffer)); // Clear the buffer. memset(IPLineBuffer, 0, sizeof(IPLineBuffer)); // Clear the buffer.
ListFile.getline(IPLineBuffer, SafeBufferSize); // Read the line. (safely NULL terminated) ListFile.getline(IPLineBuffer, SafeBufferSize); // Read the line. (safely NULL terminated)

+ 10
- 17
GBUdb.hpp View File

//////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////
// Include M_GBUdb Only Once // Include M_GBUdb Only Once


#ifndef M_GBUdb
#define M_GBUdb
#pragma once


#include "../CodeDweller/faults.hpp"
#include "../CodeDweller/faults.hpp"
#include "../CodeDweller/threading.hpp" #include "../CodeDweller/threading.hpp"
#include <cmath> #include <cmath>
#include <cctype> #include <cctype>
#include <cstdlib> #include <cstdlib>
#include <ctime> #include <ctime>


using namespace std;
namespace cd = codedweller;


const unsigned int GBUdbFlagsMask = 0xC0000000; // Top 2 bits are the flag. const unsigned int GBUdbFlagsMask = 0xC0000000; // Top 2 bits are the flag.
const unsigned int GBUdbIgnore = 0xC0000000; // Ignore is the 11 flag. const unsigned int GBUdbIgnore = 0xC0000000; // Ignore is the 11 flag.
char UTC[UTCBufferSize]; // Time stamp for this alert. char UTC[UTCBufferSize]; // Time stamp for this alert.
unsigned int IP; // IP for this alert. unsigned int IP; // IP for this alert.
GBUdbRecord R; // GBUdbRecord for this alert. GBUdbRecord R; // GBUdbRecord for this alert.
string toXML(); // Convert to an xml representation.
std::string toXML(); // Convert to an xml representation.
}; };


// Mass update kinds of operations are handled by providing a functor // Mass update kinds of operations are handled by providing a functor
private: private:
GBUdbRecord* DataArray; // Array of GBUdbRecords, nodes, etc. GBUdbRecord* DataArray; // Array of GBUdbRecords, nodes, etc.
int MyArraySize; // The size of the array in records. int MyArraySize; // The size of the array in records.
string MyFileName; // CString for the file name.
std::string MyFileName; // CString for the file name.


GBUdbIndex ixIPCount(); // Index of the IP count for this db. GBUdbIndex ixIPCount(); // Index of the IP count for this db.
GBUdbIndex ixNextFreeNode(); // Index of the Next Free Node Index. GBUdbIndex ixNextFreeNode(); // Index of the Next Free Node Index.
class GBUdb { class GBUdb {
private: private:


Mutex MyMutex; // Data sync mutex.
Mutex AlertsMutex; // Mutex for the alerts list.
cd::Mutex MyMutex; // Data sync mutex.
cd::Mutex AlertsMutex; // Mutex for the alerts list.
GBUdbDataset* MyDataset; // Array of records. GBUdbDataset* MyDataset; // Array of records.
int PostsCounter; // Counts good/bad posts. int PostsCounter; // Counts good/bad posts.


list<GBUdbAlert> MyAlerts; // Allerts list.
std::list<GBUdbAlert> MyAlerts; // Allerts list.
void recordAlertFor(unsigned int IP, GBUdbRecord& R, unsigned int C); // Append an alert record if needed. void recordAlertFor(unsigned int IP, GBUdbRecord& R, unsigned int C); // Append an alert record if needed.


public: public:


int readIgnoreList(const char* FileName = "GBUdbIgnoreList.txt"); // setIgnore for a list of IPs int readIgnoreList(const char* FileName = "GBUdbIgnoreList.txt"); // setIgnore for a list of IPs


void GetAlerts(list<GBUdbAlert>& ListToFill); // Get all current alerts & clear.
void ImportAlerts(list<GBUdbAlert>& PeerAlerts); // Default log2 alert import function.
void GetAlerts(std::list<GBUdbAlert>& ListToFill); // Get all current alerts & clear.
void ImportAlerts(std::list<GBUdbAlert>& PeerAlerts); // Default log2 alert import function.


int IPCount(); // Number of IPs stored. int IPCount(); // Number of IPs stored.
int Size(); // Size of GBUdb in bytes. int Size(); // Size of GBUdb in bytes.


}; };


//// Include inline method definitions /////////////////////////////////////////

#include "GBUdb.inline.hpp"

#endif

// End of GBUdb Include Only Once // End of GBUdb Include Only Once
//////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////

+ 0
- 354
GBUdb.inline.hpp View File

// GBUdb.inline.hpp
//
// (C) Copyright 2006 - 2009 ARM Research Labs, LLC
// See www.armresearch.com for the copyright terms.
//
// See GBUdb.hpp for details & notes.
// This file contains inline implementations.

//// GBUdbRecord Implementations ///////////////////////////////////////////////

inline GBUdbRecord::GBUdbRecord() : // Initialize a new GBUdbRecord
RawData(0) { // to ZERO.
}

inline GBUdbFlag GBUdbRecord::Flag() { // Return the flags.
return (GBUdbFlag) (RawData & GBUdbFlagsMask); // Isolate the flags from the data & return.
}

inline GBUdbFlag GBUdbRecord::Flag(GBUdbFlag f) { // Set the flags.
RawData = RawData & (~GBUdbFlagsMask); // Strip the current flags from RawData.
RawData = RawData | f; // Put the new flags into RawData.
return (GBUdbFlag) (RawData & GBUdbFlagsMask); // Return the flags now in RawData.
}

inline unsigned int GBUdbRecord::Good() { // Return the Good count.
return ((RawData & GBUdbGoodMask) >> GBUdbGoodShift); // Isolate & shift the good count, return.
}

inline unsigned int GBUdbRecord::Good(unsigned int g) { // Set the good count.
RawData = RawData & (~GBUdbGoodMask); // Strip the current good count.
g = g & GBUdbLimit; // Make g safe (within bitfield limit).
RawData = RawData | (g << GBUdbGoodShift); // Shift & combine g with RawData.
return g; // Return the safe g value.
}

inline unsigned int GBUdbRecord::Bad() { // Get the bad count.
return (RawData & GBUdbBadMask); // Isolate the bad data and return.
}

inline unsigned int GBUdbRecord::Bad(unsigned int b) { // Set the bad count.
RawData = RawData & (~GBUdbBadMask); // Strip out the current bad count.
b = b & GBUdbLimit; // Make b safe (strip any extra bits).
RawData = RawData | b; // Combine RawData with the safe b.
return b; // return the safe b.
}

inline unsigned int GBUdbRecord::addGood(unsigned int g) { // Add to the good count & normalize.
unsigned int G = Good(); // Get the good.
unsigned int B = Bad(); // Get the bad.
G = G + g; // Add the new g to the good.
while(G > GBUdbLimit) { // If normalization is required
G = G >> 1; // then reduce the new good
B = B >> 1; // and bad counts by half
} // until things are normalized.
Good(G); // Then go ahead and set the
Bad(B); // new value(s) into place.
return G; // Return the new good count.
}

inline unsigned int GBUdbRecord::addBad(unsigned int b) { // Add to the bad count & normalize.
unsigned int G = Good(); // Get the good.
unsigned int B = Bad(); // Get the bad.
B = B + b; // Add the new b to the bad.
while(B > GBUdbLimit) { // If normalization is required
G = G >> 1; // then reduce the new good
B = B >> 1; // and bad counts by half
} // until things are normalized.
Good(G); // Then go ahead and set the
Bad(B); // new value(s) into place.
return B; // Return the new good count.
}

inline GBUdbRecord& GBUdbRecord::integrate(GBUdbRecord& A, int LocalWeight, int RemoteWeight) { // Integrate A

unsigned int Gl = Good(); // Get the good and
unsigned int Bl = Bad(); // bad counts from
unsigned int Gr = A.Good(); // the local and
unsigned int Br = A.Bad(); // remote records.

Gl = (Gl * LocalWeight) + (Gr * RemoteWeight); // Combine the Good and
Bl = (Bl * LocalWeight) + (Br * RemoteWeight); // bad counts using the weights.

while(Gl > GBUdbLimit || Bl > GBUdbLimit) { // Normalize the counts by
Gl = Gl >> 1; // dividing both in half until
Bl = Bl >> 1; // they are both within limits.
}
Good(Gl); // Then set the new Good
Bad(Bl); // and bad values and return
return *this; // this object.
}

inline GBUdbIndex GBUdbRecord::Index() { // Read the record as an index.
return (GBUdbIndex) RawData;
}

inline GBUdbIndex GBUdbRecord::Index(GBUdbIndex i) { // Write the index value of the record.
RawData = (unsigned int) i;
return (GBUdbIndex) RawData;
}

// Probability is about the ratio of a given event to the total events.
// In this case, positive probabilities indicate a tendency toward spam and
// negative probabilities indicate a tendency toward ham.

inline double GBUdbRecord::Probability() { // Calculate the probability of spam
unsigned int G = Good(); // Get the good and
unsigned int B = Bad(); // bad counts and
double P = 0.0; // grab a double to hold P.
if(0 == B + G) { // If we have no counts yet
return P; // then return a zero probability.
} // If we have counts lets do the math.
P = ((double) B - (double) G) / ((double) B + (double) G); // Calculate the differential
return P; // probability and return it.
}

// The confidence we have in a probability is related to the number of samples
// that are present. We calculate the confidence on a logarithmic scale between
// one sample and half the maximum number by category (good or bad) because
// during condensation all counts may be reduced by half. That is, a 100%
// confidence is achieved when a record contains a total of half the maximum
// number of counts for a single category.

inline double GBUdbRecord::Confidence() { // Calculate our confidence in prob.
unsigned int Total = Good() + Bad(); // What is our total count of samples.
if(0 == Total) return 0.0; // No samples is no confidence.
double Confidence = (log((double)Total) / log((double)(GBUdbLimit/2))); // Calculate on a log scale.
if(1.0 < Confidence) Confidence = 1.0; // Max confidence is 1.0.
return Confidence; // Return the result.
}

//// GBUdbDataSet Inline Methods ///////////////////////////////////////////////

inline GBUdbIndex GBUdbDataset::ixIPCount() { // Index of the IP count for this db.
return MyArraySize + GBUdbIPCountOffset; // Return the offest from the end.
}

inline GBUdbIndex GBUdbDataset::ixNextFreeNode() { // Index of the Next Free Node.
return MyArraySize + GBUdbNextFreeNodeOffset; // Return the offset from the end.
}

inline GBUdbIndex GBUdbDataset::newNodeRoot() { // Allocates a new node, returns offset.
if(0 >= FreeNodes()) { // Check that we have free nodes to
throw NoFreeNodes(); // allocate. If we don't then throw!
}
GBUdbIndex NewNode = DataArray[ixNextFreeNode()].Index(); // Grab the next new node index.
DataArray[ixNextFreeNode()].Index(NewNode + GBUdbRecordsPerNode); // Move the allocator up a node.
return NewNode; // Return the allocated node.
}

inline int GBUdbDataset::ArraySize() { // Return the current Array Size.
return MyArraySize;
}

inline int GBUdbDataset::FreeNodes() { // Return the number of free nodes.
int FreeRecords = MyArraySize - DataArray[ixNextFreeNode()].RawData; // Find the number of records left.
int FreeNodes = (FreeRecords / GBUdbRecordsPerNode) - 1; // Convert to nodes and subtract the
return FreeNodes; // control node, the return the value.
}

inline int GBUdbDataset::IPCount() { // Return the IP count.
return DataArray[ixIPCount()].RawData;
}

inline int GBUdbDataset::increaseIPCount() { // When we add an IP to the db.
return DataArray[ixIPCount()].RawData++; // Increment and return the IP count.
}

inline int GBUdbDataset::decreaseIPCount() { // When we drop an IP from the db.
return DataArray[ixIPCount()].RawData--; // Decrement and return the IP count.
}

inline const char* GBUdbDataset::FileName() { // get the file name.
return MyFileName.c_str();
}

inline unsigned int GBUdbDataset::EncodedMatch(unsigned int IP) { // Encode an IP as a MatchRecord header.
return GBUdbMatchEntryBit | (IP & GBUdbMatchDataMask); // Use the MatchEntery bit and as much
} // of the remaining IP data as possible.

inline bool GBUdbDataset::isMatch(GBUdbIndex I) { // True if record at I is a match record.
return (0 != (DataArray[I].RawData & GBUdbMatchEntryBit)); // Get the raw data and check for the bit.
}

inline bool GBUdbDataset::isMatch(GBUdbIndex I, unsigned int IP) { // True if record at I is a match for IP.
return (DataArray[I].RawData == EncodedMatch(IP));
}

inline GBUdbRecord& GBUdbDataset::MatchedData(GBUdbIndex I) { // Returns the data for the match at I.
return DataArray[I + 1]; // Since I points to the match record we
} // return the record immedately after it.

inline GBUdbRecord& GBUdbDataset::SafeUnknownRecord() { // Clears and returns the Safe record.
MySafeUnknownRecord.RawData = GBUdbUnknown; // Clear the SafeUnknownRecord and
return MySafeUnknownRecord; // return it as the result.
}

inline GBUdbIndex GBUdbDataset::ixMatchListRoot() { // Index of the Match List Root Index.
return MyArraySize + GBUdbMatchListOffset;
}

inline void GBUdbDataset::increaseIPCountIfNew(GBUdbRecord& R) { // If R is GBUdbUnknown, IncreaseIPCount.
if(GBUdbUnknown == R.RawData) { increaseIPCount(); } // If new, increase the IP count.
}

inline unsigned int GBUdbDataset::remapIP00toFF(unsigned int IP) { // Remaps final octet 00 to FF if needed.
const int LowOctetMask = 0x000000FF; // Mask for seeing the low octet.
if(0 == (IP & LowOctetMask)) { // If the lowest octet is 00 then
return (IP | LowOctetMask); // change it to FF and return.
} // If the lowest octet is something else
return IP; // then return the IP as is.
}

inline void GBUdbDataset::deleteMatchAt(GBUdbIndex I) { // Recalls MatchRecord at I for reuse.
GBUdbIndex Next = DataArray[ixMatchListRoot()].Index(); // Find the current allocation list root.
DataArray[I].RawData = (Next | GBUdbMatchUnusedBit); // Point the current match to that root.
DataArray[I+1].RawData = GBUdbUnknown; // Clean out any data the match had.
DataArray[ixMatchListRoot()].Index(I); // Make this record the list root.
}

//// GBUdb Implementations /////////////////////////////////////////////////////

inline GBUdb::GBUdb() : // Construct the db as new.
PostsCounter(0) { // No posts yet.
MyDataset = new GBUdbDataset(NULL); // Construct with no file name.
}

inline GBUdb::GBUdb(const char* FileName) : // Construct the db from a file.
PostsCounter(0) { // No Posts yet.
MyDataset = new GBUdbDataset(FileName); // Load the data set by name.
}

inline GBUdb::~GBUdb() { // Destroy the db object.
if(NULL != MyDataset) { // Save first if we can.
MyDataset->save();
delete MyDataset;
}
}

inline const char* GBUdb::FileName() { // Return the file name.
return MyDataset->FileName();
}

inline const char* GBUdb::FileName(const char* NewName) { // Set/Change the file name.
return MyDataset->FileName(NewName);
}

inline void GBUdb::save() { // Save the data.
ScopeMutex JustMe(MyMutex); // Lock the mutex during this operation.
MyDataset->save(); // Save the dataset.
PostsCounter = 0; // Reset the posts counter.
}

inline void GBUdb::load() { // Load the data.
ScopeMutex JustMe(MyMutex); // Lock the mutex during this operation.
MyDataset->load(); // Load the dataset.
}

inline GBUdbRecord GBUdb::addGood(unsigned int IP, int i) { // Count an IP as good.
ScopeMutex JustMe(MyMutex); // Lock the mutex during this operation.
++PostsCounter; // Count this as a post.
GBUdbRecord& X = MyDataset->invokeRecord(IP); // Invoke the record.
unsigned int C = X.addGood(i); // Add a count to the good side.
recordAlertFor(IP, X ,C); // Record an alert if required.
return X; // Return a copy for analysis.
}

inline GBUdbRecord GBUdb::addBad(unsigned int IP, int i) { // Count an IP as bad.
ScopeMutex JustMe(MyMutex); // Lock the mutex during this operation.
++PostsCounter; // Count this as a post.
GBUdbRecord& X = MyDataset->invokeRecord(IP); // Invoke the reocrd.
unsigned int C = X.addBad(i); // Add a count to the bad side.
recordAlertFor(IP, X, C); // Record an alert if required.
return X; // Return a copy for analysis.
}

inline GBUdbRecord GBUdb::setGood(unsigned int IP) { // Set the flag to Good for this IP.
ScopeMutex JustMe(MyMutex); // Lock the mutex during this operation.
GBUdbRecord& X = MyDataset->invokeRecord(IP); // Invoke the reocrd.
X.Flag(Good); // Set the Good flag.
return X; // Return a copy for analysis.
}

inline GBUdbRecord GBUdb::setBad(unsigned int IP) { // Set the flag to Bad for this IP.
ScopeMutex JustMe(MyMutex); // Lock the mutex during this operation.
GBUdbRecord& X = MyDataset->invokeRecord(IP); // Invoke the reocrd.
X.Flag(Bad); // Set the Bad flag.
return X; // Return a copy for analysis.
}

inline GBUdbRecord GBUdb::setUgly(unsigned int IP) { // Set the flag to Ugly for this IP.
ScopeMutex JustMe(MyMutex); // Lock the mutex during this operation.
GBUdbRecord& X = MyDataset->invokeRecord(IP); // Invoke the reocrd.
X.Flag(Ugly); // Set the Ugly flag.
return X; // Return a copy for analysis.
}

inline GBUdbRecord GBUdb::setIgnore(unsigned int IP) { // Set the flag to Ignore for this IP.
ScopeMutex JustMe(MyMutex); // Lock the mutex during this operation.
GBUdbRecord& X = MyDataset->invokeRecord(IP); // Invoke the reocrd.
X.Flag(Ignore); // Set the Ignore flag.
return X; // Return a copy for analysis.
}


inline GBUdbRecord GBUdb::getRecord(unsigned int IP) { // Retrieve an IP record.
ScopeMutex JustMe(MyMutex); // Lock the mutex during this operation.
GBUdbRecord& X = MyDataset->readRecord(IP); // ReadOnly the reocrd.
return X; // Return a copy for analysis.
}

inline GBUdbRecord GBUdb::setRecord(unsigned int IP, GBUdbRecord& R) { // Store an IP record.
ScopeMutex JustMe(MyMutex); // Lock the mutex during this operation.
GBUdbRecord& X = MyDataset->invokeRecord(IP); // Invoke the reocrd.
X = R; // Overwrite X with R.
return X; // Return a copy for analysis.
}

inline GBUdbRecord GBUdb::adjustCounts(unsigned int IP, GBUdbRecord& R) { // Adds counts from R to record for IP.
ScopeMutex JustMe(MyMutex); // Lock the data for this operation.
GBUdbRecord& X = MyDataset->invokeRecord(IP); // Locate the record in the data.
X.Bad(X.Bad() + R.Bad()); // Add the reflected adjustments
X.Good(X.Good() + R.Good()); // to the good and bad counts.
return X; // Return a copy for analysis.
}

inline bool GBUdb::dropRecord(unsigned int IP) { // Drop an IP record.
ScopeMutex JustMe(MyMutex); // Lock the mutex during this operation.
return MyDataset->dropRecord(IP); // Pass on this call to our dataset.
}

inline int GBUdb::IPCount() { // Number of IPs stored.
ScopeMutex JustMe(MyMutex);
return MyDataset->IPCount();
}

inline int GBUdb::Size() { // Size of GBUdb in bytes.
ScopeMutex JustMe(MyMutex); // Lock the mutex during this operation.
return MyDataset->ArraySize() * sizeof(GBUdbRecord); // Total records converted to bytes.
}

inline double GBUdb::Utilization() { // Utilization (percent).
ScopeMutex JustMe(MyMutex); // Lock the mutex during this operation.
int TotalRecords = MyDataset->ArraySize(); // Calculate the total number of records.
int FreeRecords = MyDataset->FreeNodes() * GBUdbRecordsPerNode; // Calculate the number of unused records.
int UsedRecords = TotalRecords - FreeRecords; // Calcualte the number of used records.
return // Calculate and return as double...
((double) UsedRecords) * 100.0 / // (Used Records * 100) / (TotalRecords)
((double) TotalRecords);
}

inline int GBUdb::Posts() { // Number of posts since last snapshot.
int CurrentCount = PostsCounter; // Grab the current posts count.
return CurrentCount; // Return the count we had.
}

Write Preview
Loading…
Cancel
Save