SNFMulti/snfLOGmgr.hpp

// snfLOGmgr.hpp
//
// (C) Copyright 2006 - 2020 ARM Research Labs, LLC.
// See www.armresearch.com for the copyright terms.
//
// SNF Logging and Statistics engine.

////////////////////////////////////////////////////////////////////////////////
//// Begin snfLOGmgr include only once

#pragma once

#include <list>
#include <set>
#include <string>
#include <vector>
#include <sstream>
#include <ctime>
#include <cstdio>

#include "../CodeDweller/timing.hpp"

#ifdef WIN32

// Required because threading.hpp includes windows.h.
#include <winsock2.h>

#endif

#include "../CodeDweller/threading.hpp"
#include "../CodeDweller/histogram.hpp"

#include "snf_match.h"

#include "snfCFGmgr.hpp"
#include "snfNETmgr.hpp"
#include "GBUdb.hpp"

#include "../CodeDweller/timing.hpp"
#include "../CodeDweller/threading.hpp"
#include "../CodeDweller/histogram.hpp"

namespace cd = codedweller;

class snfNETmgr;                                                                // Declare snfNETmgr
extern const char* SNF_ENGINE_VERSION;                                          // Declare the Engine Version Data

//// DiscLogger ////////////////////////////////////////////////////////////////
// Writes log files back to Disc and double buffers data to minimize contention
// and delays. So - if it takes a few milliseconds to post the log to disc, the
// application that post()s to the log does not have to wait. Write back happens
// about once per second when enabled. Files can be appended or overwritten.

class DiscLogger : private cd::Thread {                                         // Double buffered lazy writer.
  private:
    cd::Mutex BufferControlMutex;                                               // Protects buffers while swapping.
    cd::Mutex FlushMutex;                                                       // Protects flush operations.
    std::string myPath;                                                         // Where the file should be written.
    std::string BufferA;                                                        // Log data buffer A.
    std::string BufferB;                                                        // Log data buffer B.
    bool UseANotB;                                                              // Indicates the active buffer.
    bool isDirty;                                                               // True if data not yet written.
    bool isBad;                                                                 // True if last write failed.
    bool isTimeToStop;                                                          // True when shutting down.
    bool inAppendMode;                                                          // True when in append mode.
    std::string& FlushingBuffer() { return ((UseANotB)?BufferA:BufferB); }      // Returns the buffer for flushing.
    std::string& PostingBuffer()  { return ((UseANotB)?BufferB:BufferA); }      // Returns the buffer for posting.
    bool isEnabled;                                                             // True when this should run.
    void myTask();                                                              // Write back thread task.

  public:
    DiscLogger(std::string N = "UnNamed");                                      // Constructs and starts the thread.
    ~DiscLogger();                                                              // Flushes and stops the thread.

    std::string Path(const std::string PathName) {                              // Sets the file path.
        cd::ScopeMutex NewSettings(BufferControlMutex);
        myPath = PathName;
        return myPath;
    }
    std::string Path() {                                                        // Returns the file path.
        cd::ScopeMutex DontMove(BufferControlMutex);
        return myPath;
    }

    bool AppendMode(const bool AppendNotOverwrite) {                            // Sets append mode if true.
        return (inAppendMode = AppendNotOverwrite);
    }
    bool AppendMode() { return (inAppendMode); }                                // True if in append mode.

    bool OverwriteMode(const bool OverwriteNotAppend) {                         // Sets overwrite mode if true.
        return (inAppendMode = (!OverwriteNotAppend));
    }
    bool OverwriteMode() { return (!inAppendMode); }                            // True if in overwrite mode.

    void post(const std::string Input, const std::string NewPath = "");         // Post Input to log, [set path].
    void flush();                                                               // Flush right now!
    bool Bad() { return (isBad); }                                              // True if last write failed.
    bool Good() { return (!isBad); }                                            // True if not Bad();
    bool Dirty() { return (isDirty); }                                          // True if data needs to be written.
    bool Enabled(const bool MakeEnabled) { return (isEnabled = MakeEnabled); }  // Enables writing if true.
    bool Enabled() { return (isEnabled); }                                      // True if enabled.

    const static cd::ThreadType Type;                                           // The thread's type.

    const static cd::ThreadState DiscLogger_Flush;                              // Flushing state.
    const static cd::ThreadState DiscLogger_Wait;                               // Waiting state.

};

//// IPTestRecord //////////////////////////////////////////////////////////////
// Contains a complete analysis of a given IP. snf_RulebaseHandler provides a
// test facility that accepts and processes IPTestRecord objects. The calling
// process can then submit the IPTestRecord along with it's action to the
// snfLOGmgr for logging.

class IPTestRecord {                                                            // IP Analysis Record.
  public:
    cd::IP4Address IP;                                                          // The IP to be tested.
    GBUdbRecord G;                                                              // The GBUdb Record for the IP.
    snfIPRange R;                                                               // The GBUdb classification (range).
    int Code;                                                                   // Code associated with Range.
    IPTestRecord(cd::IP4Address testIP) : IP(testIP), Code(0) {}                // Construct with an IP.
};

//// snfScanData ///////////////////////////////////////////////////////////////
// Contains testing data for a message.
// It's defined here in the LOGmgr module because this is the module that must
// log and collect statistics for each scanned message. The snfScanData object
// is the standardized way each engine reports it's scan results to snfLOGmgr.

const int MaxIPsPerMessage = 50;                                                // Maximum number of IPs to scan per message.

struct IPScanRecord {                                                           // Structure for IP scan results.
    int Ordinal;                                                                // Which IP starting with zero.
    unsigned int IP;                                                            // What is the IP.
    GBUdbRecord GBUdbData;                                                      // GBUdb data.
};

class snfScanData {                                                             // Scan Data record for each message.

  private:

    IPScanRecord MyIPScanData[MaxIPsPerMessage];                                // Array of IP scan results.
    int MyIPCount;                                                              // Count of IP scan results.
    bool DrillDownFlags[MaxIPsPerMessage];                                      // DrillDown flags. (Set Ignore).

    int SourceIPOrdinal;                                                        // Ordinal to source IP scan data.
    bool SourceIPFoundFlag;                                                     // True if source IP is set.
    snfIPRange SourceIPRangeFlag;                                               // GBUdb detection range for source IP.

    cd::IP4Address myCallerForcedSourceIP;                                      // Caller forced source IP if not 0UL.
    cd::IP4Address myHeaderDirectiveSourceIP;                                   // Header forced source IP if not 0UL.

  public:

    snfScanData(int ScanHorizon);                                               // Constructor.
    ~snfScanData();                                                             // Destructor.

    // The ReadyToClear bit helps multi-phase input situations where the first
    // phase might add some input data before calling the base-level scanner.
    // In those cases, the pre-scan-phase will clear() the ScanData (and with
    // it the ReadyToClear bit) before adding a few critical pieces of data -
    // such as the scan name and the scan-start UTC for example. When the base
    // level scanner is called to perform the actual scan, the clear() call
    // will be inert so that any pre-set data will be preserved.

    bool ReadyToClear;                                                          // True when Logging is done.
    void clear();                                                               // Clear for a new message.

    class NoFreeIPScanRecords {};                                               // Thrown when we run out of scan records.
    class OutOfBounds {};                                                       // Thrown in IPScanData if no record at i.

    int IPScanCount();                                                          // Return the number of IPs.
    IPScanRecord& newIPScanRecord();                                            // Get the next free IP scan record.
    IPScanRecord& IPScanData(int i);                                            // Return the IP scan record i.

    // 20080221 _M We can now define in header directives patterns for Received
    // headers that we should drill past if they show up as a message source
    // candidate. This allows GBUdb to learn to ignore certain IPs automatically
    // as they arrive either by IP stubs such as "[12.34.56." or by reverse DNS
    // data such as "friendly.example.com [". When the header directives engine
    // scans the headers it will call drillPastOrdinal for any Received header
    // that matches a <drilldown/> directive. Later when the header analysis
    // engine tries to pick the source for the message it will check each source
    // candidate against the isDrillDownSource() method. If the source is to be
    // ignored then it will set the ignore flag for that IP, process it as if
    // it were ignored, and continue searching for the actual source.

    void drillPastOrdinal(int O);                                               // Sets Drill Down flag for IP record O.
    bool isDrillDownSource(IPScanRecord& X);                                    // True if we drill through this source.

    cd::IP4Address HeaderDirectiveSourceIP(cd::IP4Address A);                   // set Header directive source IP.
    cd::IP4Address HeaderDirectiveSourceIP();                                   // get Header directive source IP.
    cd::IP4Address CallerForcedSourceIP(cd::IP4Address A);                      // set Caller forced source IP.
    cd::IP4Address CallerForcedSourceIP();                                      // get Caller forced source IP.

    IPScanRecord& SourceIPRecord(IPScanRecord& X);                              // Sets the source IP record.
    IPScanRecord& SourceIPRecord();                                             // Gets the source IP record.
    bool FoundSourceIP();                                                       // True if the source IP record was set.
    snfIPRange SourceIPRange();                                                 // GET Source IP range.
    snfIPRange SourceIPRange(snfIPRange R);                                     // SET Source IP range for this scan.

    // Direct access data...

    std::string SourceIPEvaluation;                                             // GBUdb Source IP evaluation.

    // LogControl and General Message Flags

    time_t StartOfJobUTC;                                                       // Timestamp at start of job.
    int SetupTime;                                                              // Time in ms spent setting up to scan.
    std::string ScanName;                                                       // Identifying name or message file name.
    cd::Timer ScanTime;                                                         // Scan time in ms.
    int ScanDepth;                                                              // Scan Depth in evaluators.

    std::string ClassicLogText;                                                 // Classic log entry text if any.
    std::string XMLLogText;                                                     // XML log entry text if any.
    std::string XHDRsText;                                                      // XHeaders text if any.
    bool XHeaderInjectOn;                                                       // True if injecting headers is on.
    bool XHeaderFileOn;                                                         // True if creating .xhdr file is on.

    bool MessageFileTypeCGPOn;                                                  // Expect a CGP type message file.

    unsigned int ScanSize;                                                      // What size is the scan request.

    // GBUdb Activity Flags

    bool GBUdbNormalTriggered;                                                  // True if GBUdb indeterminate IP source.

    bool GBUdbWhiteTriggered;                                                   // True if GBUdb found source IP white.
    bool GBUdbWhiteSymbolForced;                                                // True if white was on and symbol was set.
    bool GBUdbPatternSourceConflict;                                            // True if pattern was found with white IP.
    bool GBUdbAutoPanicTriggered;                                               // True if autopanic was triggered.
    bool GBUdbAutoPanicExecuted;                                                // True if an autopanic was added.

    bool GBUdbBlackTriggered;                                                   // True if GBUdb found source IP black.
    bool GBUdbBlackSymbolForced;                                                // True if black was on and symbol was set.
    bool GBUdbTruncateTriggered;                                                // True if Truncate was possible.
    bool GBUdbPeekTriggered;                                                    // True if we could peek.
    bool GBUdbSampleTriggered;                                                  // True if we could sample.
    bool GBUdbTruncateExecuted;                                                 // True if we actually did truncate.
    bool GBUdbPeekExecuted;                                                     // True if we peeked instead of truncating.
    bool GBUdbSampleExecuted;                                                   // True if we sampled.

    bool GBUdbCautionTriggered;                                                 // True if GBUdb found source IP suspicous.
    bool GBUdbCautionSymbolForced;                                              // True if caution was on and symbol was set.

    // Rule panics

    std::set<int> RulePanics;                                                   // A list of rule IDs panicked this scan.

    // Pattern Engine Scan Result Data

    std::vector<unsigned char> FilteredData;                                    // Message data after filter chain.
    unsigned long int HeaderDirectiveFlags;                                     // Flags set by header directives.

    bool PatternWasFound;                                                       // True if the pattern engine matched.
    int PatternID;                                                              // The winning rule ID.
    int PatternSymbol;                                                          // The associated symbol.

    std::list<snf_match> MatchRecords;                                          // List of match records.
    std::list<snf_match>::iterator MatchRecordsCursor;                          // Localized iterator for match records.
    int MatchRecordsDelivered;                                                  // Match records seen so far.

    int CompositeFinalResult;                                                   // What the scan function returned.

};

//// SMHDMY counter
//
// Provides a running SUM for a series of sliding windows. The input() expects
// a new piece of data every second (or so). It is presumed that another counter
// will keep track of the actual milliseconds if accuracy is required. The object
// is all primative data parts so it is possible to store and retrieve this object
// in binary format on the same system when that's helpful.

class snf_SMHDMY_Counter {                                                      // Sliding window "live" counter.
  private:

    bool do_input(int X, int& SUM, int* DATA, int& ORDINAL, int SIZE);          // Subroutine for assimilating input.

  public:
    snf_SMHDMY_Counter() {                                                      // When making a new one, reset all
        memset(this, 0, sizeof(snf_SMHDMY_Counter));                            // data to zero. It's all ints ;-)
    }

    // 60 seconds is a minute (6 x 10)

    int SEC6DATA[6], SEC6SUM, SEC6ORDINAL;
    int SEC10DATA[10], SEC10SUM, SEC10ORDINAL;

    // 60 minutes is an hour (6 x 10)

    int MIN6DATA[6], MIN6SUM, MIN6ORDINAL;
    int MIN10DATA[10], MIN10SUM, MIN10ORDINAL;

    // 24 hours is a day (4 x 6)

    int HOUR4DATA[4], HOUR4SUM, HOUR4ORDINAL;
    int HOUR6DATA[6], HOUR6SUM, HOUR6ORDINAL;

    // 7 days is a week (7)

    int WEEK7DATA[7], WEEK7SUM, WEEK7ORDINAL;

    // 30 days is a month (5 x 6)

    int MONTH5DATA[5], MONTH5SUM, MONTH5ORDINAL;
    int MONTH6DATA[6], MONTH6SUM, MONTH6ORDINAL;

    // 12 months (almost) is a year (3 x 4)

    int YEAR3DATA[3], YEAR3SUM, YEAR3ORDINAL;
    int YEAR4DATA[4], YEAR4SUM, YEAR4ORDINAL;

    // 365 days is a year

    int YEAR365DATA[365], YEAR365SUM, YEAR365ORDINAL;

    void input(int X);                                                          // Add new data to the counter.

    bool Cycled60Seconds() { return (0 == SEC6ORDINAL && 0 == SEC10ORDINAL); }  // Full cycle of data for seconds.
    int Sum60Seconds() { return SEC10SUM; }
    int Sum66Seconds() { return (SEC6SUM + SEC10SUM); }
    int SumThru1Minute() { return Sum66Seconds(); }                             // All samples thru one minute.

    bool Cycled60Minutes() {                                                    // Full cycle of data for minutes.
        return (Cycled60Seconds() && 0 == MIN6ORDINAL && 0 == MIN10ORDINAL);
    }

    int Sum60Minutes() { return MIN10SUM; }
    int Sum66Minutes() { return (MIN6SUM + MIN10SUM); }
    int SumThru1Hour() { return SumThru1Minute() + Sum66Minutes(); }            // All samples thru one hour.

    bool Cycled24Hours() {                                                      // Full cycle of data for hours.
        return (Cycled60Minutes() && 0 == HOUR4ORDINAL && 0 == HOUR6ORDINAL);
    }

    int Sum24Hours() { return HOUR6SUM; }
    int Sum28Hours() { return (HOUR4SUM + HOUR6SUM); }
    int SumThru1Day() { return SumThru1Hour() + Sum28Hours(); }                 // All samples thru one day.

    bool Cycled7Days() { return (Cycled24Hours() && 0 == WEEK7ORDINAL); }       // Full cycle of data for week.

    int Sum7Days() { return WEEK7SUM; }
    int SumThru1Week() { return SumThru1Day() + Sum7Days(); }                   // All samples thru one week.

    bool Cycled30Days() {                                                       // Full cycle of data for month.
        return (Cycled24Hours() && 0 == MONTH6ORDINAL && 0 == MONTH5ORDINAL);
    }

    int Sum30Days() { return MONTH6SUM; }
    int Sum35Days() { return (MONTH5SUM + MONTH6SUM); }
    int SumThru1Month() { return SumThru1Day() + Sum35Days(); }                  // All samples thu one month.

    bool Cycled12Months() {                                                     // Full cycle of data for 12 months.
        return (Cycled30Days() && 0 == YEAR3ORDINAL && 0 == YEAR4ORDINAL);
    }

    int Sum450Days() { return (YEAR3SUM + YEAR4SUM); }
    int SumThru1Year() { return SumThru1Month() + Sum450Days(); }               // All samples thru one year.

    bool Cycled365Days() { return (Cycled24Hours() && 0 == YEAR365ORDINAL); }   // Full cycle of data for 365 days.

    int Sum365Days() { return YEAR365SUM; }

};

//// snfLOGmgr /////////////////////////////////////////////////////////////////

// A note about the LOG manager and configuration data:
// Events that are logged with the log manager may come from scans using
// different configurations. In order to keep things as sane as possible,
// operations that are dependent on configuration information such as creating
// log file entries or producing status page data will require that an
// appropriate snfCFGData object be provided by reference and that the
// snfCFGData object be guaranteed to remain stable for the duration of the
// call. Changing snfCFGData may result in inconsistent results.
//
// This requirement is fairly easy to accomplish since posts to the LOGmgr
// will come from scanning engines that have a snfCFGPacket "grab()ed" during
// their operations, and executive requests will come from the ruelbase
// manager which can grab a snfCFGPacket for the duration of the request.

const int NumberOfResultCodes = 64;

class snfCounterPack {
  public:
    snfCounterPack();                                                           // Construct new CounterPacks clean.
    void reset();                                                               // How to reset a counter pack.

    cd::Timer ActiveTime;                                                       // Measures Active (swapped in) Time.

    struct {

        unsigned long Scans;                                                    // Number of messages scanned.
        unsigned long Spam;                                                     // Count of spam results.
        unsigned long Ham;                                                      // Count of ham results.

        unsigned long GBUdbNormalTriggered;                                     // Count of indeterminate gbudb IP hits.

        unsigned long GBUdbWhiteTriggered;                                      // Count of GBUdb found source IP white.
        unsigned long GBUdbWhiteSymbolForced;                                   // Count of white was on and symbol was set.
        unsigned long GBUdbPatternSourceConflict;                               // Count of pattern was found with white IP.
        unsigned long GBUdbAutoPanicTriggered;                                  // Count of autopanic was triggered.
        unsigned long GBUdbAutoPanicExecuted;                                   // Count of an autopanic was added.

        unsigned long GBUdbBlackTriggered;                                      // Count of GBUdb found source IP black.
        unsigned long GBUdbBlackSymbolForced;                                   // Count of black was on and symbol was set.
        unsigned long GBUdbTruncateTriggered;                                   // Count of Truncate was possible.
        unsigned long GBUdbPeekTriggered;                                       // Count of we could peek.
        unsigned long GBUdbSampleTriggered;                                     // Count of we could sample.
        unsigned long GBUdbTruncateExecuted;                                    // Count of if we actually did truncate.
        unsigned long GBUdbPeekExecuted;                                        // Count of we peeked instead of truncating.
        unsigned long GBUdbSampleExecuted;                                      // Count of we sampled.

        unsigned long GBUdbCautionTriggered;                                    // Count of GBUdb found source IP suspicous.
        unsigned long GBUdbCautionSymbolForced;                                 // Count of caution was on and symbol was set.

        unsigned long PatternWasFound;                                          // Count of scanner matches.

        unsigned long RulePanicFound;                                           // Count of rule panics.

    } Events;
};

//// Interval timers precisely track the time between hack()s. There are
//// two timers inside. One is active, the other is stopped. Each time hack()
//// is called, one timer becomes active at the moment the other is stopped.

class IntervalTimer {                                                           // Precision interval timer.

  private:

    cd::Timer A;                                                                // Here is one timer.
    cd::Timer B;                                                                // Here is the other timer.
    bool ANotB;                                                                 // True if A is the active timer.

    cd::Timer& Active();                                                        // Selects the active timer.
    cd::Timer& Inactive();                                                      // Selects the inactive timer.

  public:

    cd::msclock hack();                                                         // Chop off a new interval & return it.
    cd::msclock Interval();                                                     // Return the last interval.
    cd::msclock Elapsed();                                                      // Return the time since last hack.
};

//// PersistentState stores the counters we keep between runs.

class snfLOGPersistentState {
  public:

    snfLOGPersistentState() :
      Ready(0),
      LastSyncTime(0),
      LastSaveTime(0),
      LastCondenseTime(0),
      LatestRuleID(0),
      SerialNumberCounter(0)  {}

    bool Ready;                                                                 // True if we're ready to use.

    void store(std::string& FileNameToStore);                                   // Write the whole thing to a file.
    void restore(std::string& FileNameToRestore);                               // Read the whole thing from a file.

    time_t LastSyncTime;                                                        // time_t of last Sync event.
    time_t LastSaveTime;                                                        // time_t of last GBUdb Save event.
    time_t LastCondenseTime;                                                    // time_t of last GBUdb Condense event.

    int LatestRuleID;                                                           // Latest rule ID seen so far.
    int SerialNumberCounter;                                                    // Remembers the serial number.
};

class snfLOGmgr : private cd::Thread {

  private:

    cd::Mutex MyMutex;                                                          // Mutex to serialize updates & queries.
    cd::Mutex ConfigMutex;                                                      // Mutex to protect config changes.
    cd::Mutex SerialNumberMutex;                                                // Protects the serial number.
    cd::Mutex PeekMutex;                                                        // Protects Peek Loop Counter.
    cd::Mutex SampleMutex;                                                      // Protects Sample Loop Counter.
    cd::Mutex StatusReportMutex;                                                // Protects status report post & get.

    snfCounterPack CounterPackA, CounterPackB;                                  // Swapable counter packs.

    snfCounterPack* CurrentCounters;                                            // Current Event Counters.
    snfCounterPack* ReportingCounters;                                          // Counters being used to collect data.

    snfCounterPack* getSnapshot();                                              // Get a copy of the current counters.

    volatile bool Configured;                                                   // True if we're properly configured.
    volatile bool TimeToDie;                                                    // True when the thread should stop.

    volatile int PeekEnableCounter;                                             // How many peek attempts recently?
    volatile int SampleEnableCounter;                                           // How many sample attempts recently?

    void myTask();                                                              // Thread task.

    time_t StartupTime;                                                         // Time since engine started.

    snfLOGPersistentState Status;                                               // Persistent State Data.
    std::string PersistentFileName;                                             // File name for the State Data.

    snfNETmgr* myNETmgr;                                                        // Net manager link.
    GBUdb* myGBUdb;                                                             // GBUdb link.

    // Configuration

    std::string ActiveRulebaseUTC;                                              // UTC of last successful load.
    std::string AvailableRulebaseUTC;                                           // UTC of rulebase available for update.
    bool NewerRulebaseIsAvailable;                                              // True if a newer rulebase is available.

    std::string myPlatformVersion;                                              // Version info for platform.

    bool Rotate_LocalTime;                                                      // Rotate logs using localtime.

    std::string LogsPath;                                                       // Path to logs directory.
    bool ClassicLogRotate;                                                      // True = Rotate Classic Log.
    bool XMLLogRotate;                                                          // True = Rotate XML Log.

    // Live stats

    snf_SMHDMY_Counter MessageCounter;
    snf_SMHDMY_Counter HamCounter;
    snf_SMHDMY_Counter SpamCounter;
    snf_SMHDMY_Counter WhiteCounter;
    snf_SMHDMY_Counter CautionCounter;
    snf_SMHDMY_Counter BlackCounter;
    snf_SMHDMY_Counter TruncateCounter;
    snf_SMHDMY_Counter SampleCounter;
    snf_SMHDMY_Counter AutoPanicCounter;
    snf_SMHDMY_Counter RulePanicCounter;
    snf_SMHDMY_Counter TimeCounter;

    // Histograms

    cd::Histogram ResultsSecond;
    cd::Histogram ResultsMinute;
    cd::Histogram ResultsHour;
    cd::Histogram RulesSecond;
    cd::Histogram RulesMinute;
    cd::Histogram RulesHour;
    cd::Histogram PanicsSecond;
    cd::Histogram PanicsMinute;
    cd::Histogram PanicsHour;

    // Reporting

    std::string NodeId;                                                         // We need this for our status msgs.
    void do_StatusReports();                                                    // Update & sequence status reports.

    int XML_Log_Mode;                                                           // What is the XML log mode.
    int Classic_Log_Mode;                                                       // What is the Classic log mode.

    // Every second we get the basics and collect data. (local only)

    bool SecondReport_Log_OnOff;
    bool SecondReport_Append_OnOff;
    std::string SecondReport_Log_Filename;
    std::string SecondReportText;
    std::string SecondReportTimestamp;
    bool do_SecondReport();                                                     // Send our 1 second status report.

    // Every minute we get hard data and event logs. (for sync)

    bool MinuteReport_Log_OnOff;
    bool MinuteReport_Append_OnOff;
    std::string MinuteReport_Log_Filename;
    std::string MinuteReportText;
    std::string MinuteReportTimestamp;
    cd::Histogram PatternRulesHistogram;
    bool do_MinuteReport();                                                     // Send our 1 minute status report.

    // Every hour we get a summary.

    bool HourReport_Log_OnOff;
    bool HourReport_Append_OnOff;
    std::string HourReport_Log_Filename;
    std::string HourReportText;
    std::string HourReportTimestamp;
    bool do_HourReport();                                                       // Send our 1 hour status report.

    void postStatusLog(                                                         // Post a Status log if required.
      const std::string& LogData,                                               // Here's the log entry's data.
      const std::string& LogFileName,                                           // Here is where it should go.
      const bool LogEnabled,                                                    // This is true if we should write it.
      const bool AppendNotOverwrite,                                            // True=Append, False=Overwrite.
      DiscLogger& Logger                                                        // Lazy Log Writer to use.
    );

    DiscLogger SecondStatusLogger;                                              // Lazy writer for Second status.
    DiscLogger MinuteStatusLogger;                                              // Lazy writer for Minute status.
    DiscLogger HourStatusLogger;                                                // Lazy writer for Hour status.
    DiscLogger XMLScanLogger;                                                   // Lazy writer for XML Scan log.
    DiscLogger ClassicScanLogger;                                               // Lazy writer for Classic Scan log.

    void doXHDRs(snfCFGData& CFGData, snfScanData& ScanData);                   // XHDR sub routine for LogThisScan()
    void doXMLLogs(snfCFGData& CFGData, snfScanData& ScanData);                 // XML sub routine for LogThisScan()
    void doClassicLogs(snfCFGData& CFGData, snfScanData& ScanData);             // Classic sub routine for LogThisScan()

    void captureLTSMetrics(snfCFGData& CFGData, snfScanData& ScanData);         // LogThisScan section 1, Locked.
    void performLTSLogging(snfCFGData& CFGData, snfScanData& ScanData);         // LogThisScan section 2, Unlocked.

  public:

    snfLOGmgr();                                                                // Initialize & start the thread.
    ~snfLOGmgr();                                                               // Stop the thread & clean up.

    void stop();                                                                // Stops the manager.

    void linkNETmgr(snfNETmgr& N);                                              // Link in my NETmgr
    void linkGBUdb(GBUdb& G);                                                   // Link in my GBUdb

    void configure(snfCFGData& CFGData);                                        // Update the configuration.

    void updateActiveUTC(std::string ActiveUTC);                                // Set active rulebase UTC.

    void logThisIPTest(IPTestRecord& I, std::string Action);                    // Capthre the data from an IP test.

    void logThisScan(snfCFGData& CFGData, snfScanData& ScanData);               // Capture the data from this scan.

    void logThisError(snfScanData& ScanData, const std::string ContextName,     // Inject an error log entry for this
      const int Code, const std::string Text                                    // scan using this number & message.
    );

    void logThisError(std::string ContextName, int Code, std::string Text);     // Log an error message.

    void logThisInfo(std::string ContextName, int Code, std::string text);      // Log an informational message.

    std::string PlatformVersion(std::string NewPlatformVersion);                // Set platform version info.
    std::string PlatformVersion();                                              // Get platform version info.

    std::string EngineVersion();                                                // Get engine version info.

    void updateAvailableUTC(std::string& AvailableRulebaseTimestamp);           // Stores Available, true==update ready.
    std::string ActiveRulebaseTimestamp();                                      // Get active rulebase timestamp.
    std::string AvailableRulebaseTimestamp();                                   // Get available rulebase timestamp.
    bool isUpdateAvailable();                                                   // True if update is available.


    bool OkToPeek(int PeekOneInX);                                              // Check to see if it's ok to peek.
    bool OkToSample(int SampleOneInX);                                          // Check to see if it's ok to sample.

    time_t Timestamp();                                                         // Get an ordinary timestamp.
    std::string Timestamp(time_t t);                                            // Convert time_t to a timestamp s.
    std::string& Timestamp(std::string& s);                                     // Appends a current timestamp in s.
    std::string LocalTimestamp(time_t t);                                       // Convert time_t to a local timestamp s.
    std::string& LocalTimestamp(std::string& s);                                // Appends a current local timestamp in s.
    unsigned int SerialNumber();                                                // Returns the next serial number.
    std::string& SerialNumber(std::string& s);                                  // Appends the next serial number.

    int SecsSinceStartup();                                                     // Gets seconds since starup.
    void RecordSyncEvent();                                                     // Sets timestamp of latest Sync.
    int SecsSinceLastSync();                                                    // Gets seconds since latest Sync.
    void RecordSaveEvent();                                                     // Sets timestamp of latest Save.
    int SecsSinceLastSave();                                                    // Gets seconds since latest Save.
    void RecordCondenseEvent();                                                 // Sets timestamp of latest Condense.
    int SecsSinceLastCondense();                                                // Gets seconds since latest Condense.

    // Live stats functions

    double MessagesPerMinute();                                                 // Avg Msgs/Minute.
    double HamPerMinute();                                                      // Avg Ham/Minute.
    double SpamPerMinute();                                                     // Avg Spam/Minute.
    double WhitePerMinute();                                                    // Avg White/Minute.
    double CautionPerMinute();                                                  // Avg Caution/Minute.
    double BlackPerMinute();                                                    // Avg Black/Minute.
    double TruncatePerMinute();                                                 // Avg Truncate/Minute.
    double SamplePerMinute();                                                   // Avg Sample/Minute.
    int LatestRuleID();                                                         // Returns the latest Rule ID seen.

    int RunningTime();                                                          // Seconds running since startup.

    std::string getStatusSecondReport();                                        // Get latest status.second report.
    std::string getStatusMinuteReport();                                        // Get latest status.minute report.
    std::string getStatusHourReport();                                          // Get latest status.hour report.

    const static cd::ThreadType Type;                                           // The thread's type.

};

//// End snfLOGmgr include only once
////////////////////////////////////////////////////////////////////////////////