Created Misc directory

Populated Misc directory with architectural notes
Populated Misc directory with a sample configuration

git-svn-id: https://svn.microneil.com/svn/SNF4CGP/trunk@32 59e8e3e7-56fa-483b-b4b4-fa6ab0af3dfc

Misc/GBUdbIgnoreList.txt

@@ -0,0 +1,9 @@
+# List of IPs to Ignore on startup
+# Each IP in this list is set to Ignore in GBUdb when
+# The configuration is loaded.
+# Hash mark on the beginning of a line indicates a comment.
+# Comments after an IP are also ignored.
+# One line per IP. Sorry, no CIDR yet.
+# Be sure to list ALL of your gateways :-)
+
+127.0.0.1	# ignore localhost, of course.

Misc/Notes.txt

@@ -0,0 +1,61 @@
+20091009.1549 _M
+
+Copyright (C) 2009 ARM Research Labs, LLC
+
+SNF4CGP provides a high-performance, multi-threaded spam filter plugin for
+Communigate Pro using the Message Sniffer (SNFMulti) engine.
+
+http://www.communigate.com/CommuniGatePro/VirusScan.html
+http://www.communigate.com/CommuniGatePro/Helpers.html
+http://www.communigate.com/CommuniGatePro/Data.html#String
+
+The general design goals are (in no particular order):
+
+1. Provide a customizable match between each possible SNF result code and
+   each possible communigate response.
+
+2. Provide comprehensive debugging and performance logging capabilities
+   consistent with the way Communigate operates.
+
+3. Provide a flexible message-hold capability to allow for additional processing
+   by other software for quarantine operations or further message handing.
+
+4. Provide the highest possible message handling performance.
+
+5. Create an architecture that is easy to extend and support on multiple
+   platforms.
+
+See Architecture.jpg for a drawing of the architecture. Note that the snap-shot
+is slightly out of spec. When the Architecture.jpg was snapped the thinking was
+that each Job in the pool would contain it's own dedicated scanner. Later during
+development it was decided that scanners should also be pooled and conserved and
+only bound to a job for as short a time as possible. The latest drawing shows
+this by connecting a Scanner Pool to the scan operation with a short loop and
+indicating that the scanner pool resides inside the Job pool with a small symbol
+in each. When we redraw this architectural diagram we will include these changes.
+
+The application uses threads in this way:
+
+* One thread sets up the application, reads requests, and ultimately closes
+down the application.
+
+* Another thread processes all finished jobs by pumping their output through
+cout and then recycling job objects.
+
+* Worker threads from a pool process each request.
+
+Resource contention is minimized wherever possible:
+
+- Ram allocation is conserved by pre-allocating and reusing buffers.
+
+- Scanners, Worker threads, and Jobs are allocated as needed and pooled.
+
+- Scanners are borrowed from a pool only long enough to scan a message and
+  capture the results.
+
+- Configuration data is parsed and cached only when it has changed.
+
+- Worker threads are released back to the pool without waiting for IO to cout.
+
+- As much as possible thread interactions are reduced to reference passing
+  in very short mutex protected functions.

Misc/identity.xml

@@ -0,0 +1,2 @@
+<snf><identity licenseid='licensid' authentication='authenticationxx'/></snf>
+

Misc/snf_engine.xml

@@ -0,0 +1,162 @@
+<!-- SNFMulti V3.0 Configuration File, Setup: Typical of SNF4CGP -->
+<!-- http://www.armresearch.com/support/articles/software/snfServer/config/snfEngine.jsp -->
+
+<snf>
+    <node identity='C:\m\Projects\MessageSniffer\SNF4CGP_Work\TestEnvironment\identity.xml'>
+
+        <paths>
+            <log path='C:\m\Projects\MessageSniffer\SNF4CGP_Work\TestEnvironment\'/>
+            <rulebase path='C:\m\Projects\MessageSniffer\SNF4CGP_Work\TestEnvironment\'/>
+            <workspace path='C:\m\Projects\MessageSniffer\SNF4CGP_Work\TestEnvironment\'/>
+        </paths>
+
+        <logs>
+
+            <rotation localtime='no'/>
+
+            <status>
+                <second log='yes' append='no'/>
+                <minute log='yes' append='no'/>
+                <hour log='no' append='no'/>
+            </status>
+
+            <scan>
+                <identifier force-message-id='no'/>
+                <classic mode='api' rotate='yes' matches='unique'/>
+                <xml mode='file' rotate='yes' matches='all' performance='yes' gbudb='yes'/>
+
+                <xheaders>
+                    <output mode='api'/>
+
+                    <version on-off='off'>X-MessageSniffer-Version</version>
+                    <license on-off='off'>X-MessageSniffer-License</license>
+                    <rulebase on-off='off'>X-MessageSniffer-RulebaseUTC</rulebase>
+                    <identifier  on-off='off'>X-MessageSniffer-Identifier</identifier>
+                    <gbudb on-off='on'>X-GBUdb-Analysis</gbudb>
+                    <result on-off='on'>X-MessageSniffer-Scan-Result</result>
+                    <matches on-off='on'>X-MessageSniffer-Rules</matches>
+
+                    <black on-off='off'>X-MessageSniffer-Spam: Yes</black>
+                    <white on-off='off'>X-MessageSniffer-White: Yes</white>
+                    <clean on-off='off'>X-MessageSniffer-Clean: Yes</clean>
+                    <symbol on-off='off' n='0'>X-MessageSniffer-SNF-Group: OK</symbol>
+                    <symbol on-off='off' n='20'>X-MessageSniffer-SNF-Group: Truncated</symbol>
+                    <symbol on-off='off' n='40'>X-MessageSniffer-SNF-Group: Caution</symbol>
+                    <symbol on-off='off' n='63'>X-MessageSniffer-SNF-Group: Black</symbol>
+                    <symbol on-off='off' n='62'>X-MessageSniffer-SNF-Group: Obfuscation</symbol>
+                    <symbol on-off='off' n='61'>X-MessageSniffer-SNF-Group: Abstract</symbol>
+                    <symbol on-off='off' n='60'>X-MessageSniffer-SNF-Group: General</symbol>
+                    <symbol on-off='off' n='59'>X-MessageSniffer-SNF-Group: Casinos-Gambling</symbol>
+                    <symbol on-off='off' n='58'>X-MessageSniffer-SNF-Group: Debt-Credit</symbol>
+                    <symbol on-off='off' n='57'>X-MessageSniffer-SNF-Group: Get-Rich</symbol>
+                    <symbol on-off='off' n='56'>X-MessageSniffer-SNF-Group: Ink-Toner</symbol>
+                    <symbol on-off='off' n='55'>X-MessageSniffer-SNF-Group: Malware</symbol>
+                    <symbol on-off='off' n='54'>X-MessageSniffer-SNF-Group: Porn-Dating-Adult</symbol>
+                    <symbol on-off='off' n='53'>X-MessageSniffer-SNF-Group: Scam-Phishing</symbol>
+                    <symbol on-off='off' n='52'>X-MessageSniffer-SNF-Group: Snake-Oil</symbol>
+                    <symbol on-off='off' n='51'>X-MessageSniffer-SNF-Group: Spamware</symbol>
+                    <symbol on-off='off' n='50'>X-MessageSniffer-SNF-Group: Media-Theft</symbol>
+                    <symbol on-off='off' n='49'>X-MessageSniffer-SNF-Group: AV-Push</symbol>
+                    <symbol on-off='off' n='48'>X-MessageSniffer-SNF-Group: Insurance</symbol>
+                    <symbol on-off='off' n='47'>X-MessageSniffer-SNF-Group: Travel</symbol>
+
+                </xheaders>
+            </scan>
+        </logs>
+
+        <network>
+            <sync secs='30' host='sync.messagesniffer.net' port='25'/>
+            <update-script on-off='on' call='C:\m\Projects\MessageSniffer\SNF4CGP_Work\TestEnvironment\getRulebase.cmd' guard-time='180'/>
+        </network>
+
+        <xci on-off='on' port='9001'/>
+
+        <gbudb>
+            <database>
+
+                <condense minimum-seconds-between='600'>
+                    <time-trigger on-off='on' seconds='86400'/>
+                    <posts-trigger on-off='off' posts='1200000'/>
+                    <records-trigger on-off='off' records='600000'/>
+                    <size-trigger on-off='on' megabytes='150'/>
+                </condense>
+
+                <checkpoint on-off='on' secs='3600'/>
+
+            </database>
+
+            <regions>
+
+                <white on-off='on' symbol='0'>
+                    <edge probability='-1.0' confidence='0.4'/>
+                    <edge probability='-0.8' confidence='1.0'/>
+                    <panic on-off='on' rule-range='1000'/>
+                </white>
+
+                <caution on-off='on' symbol='40'>
+                    <edge probability='0.4' confidence='0.0'/>
+                    <edge probability='0.8' confidence='0.5'/>
+                </caution>
+
+                <black on-off='on' symbol='63'>
+                    <edge probability='0.8' confidence='0.2'/>
+                    <edge probability='0.8' confidence='1.0'/>
+                    <truncate on-off='on' probability='0.9' peek-one-in='5' symbol='20'/>
+                    <sample on-off='on' probability='0.8' grab-one-in='5' passthrough='no' passthrough-symbol='0'/>
+                </black>
+
+            </regions>
+
+            <training on-off='on'>
+
+                <bypass>
+                    <!-- <header name='To:' find='spam@example.com'/> -->
+                    <!-- <header name='Received:' ordinal='1' find='friendlyhost.com'/> -->
+                </bypass>
+
+                <drilldown>
+                    <!-- <received ordinal='0' find='[12.34.56.'/> where we want to ignore 12.34.56.0/24 -->
+                    <!-- <received ordinal='0' find='mixed-source.com'/> -->
+                    <!-- <received ordinal='1' find='mixed-source-internal.com'/> -->
+                </drilldown>
+
+                <source>
+                    <!-- <header name='X-Use-This-Source:' received='mixedsource.com [' ordinal='0' /> -->
+                    <!-- <header name='X-Originating-IP:' received='hotmail.com [' ordinal='0' /> -->
+                </source>
+
+                <white>
+                    <result code='1'/>
+                    <!-- <header name='Received:' ordinal='0' find='.friendlyhost.com'/> -->
+                </white>
+
+            </training>
+
+         </gbudb>
+
+         <rule-panics>
+             <!--
+             <rule id='123456'/>
+             <rule id='123457'/>
+             -->
+         </rule-panics>
+
+         <platform>
+             <snf4cgp>
+                 <ham action='Allow' reason='Message OK' comment='Message OK' headers='yes' xml='yes'>
+                     <result code='0' comment='(0) Not Spam/Malware' />
+                     <result code='1' comment='(1) White Rule/IP-Range' />
+                 </ham>
+                 <spam action='Allow' reason='Spam/Malware' comment='Spam/Malware' headers='yes' classic='no' xml='no' hold-path='quarantine'>
+                     <result code='40' comment='(40) Caution' />
+                     <result code='63' action='Reject' reason='Source IP black listed (GBUdb/black)' comment='(63) Black' />
+                     <result code='20' action='Reject' reason='Source IP black listed (GBUdb/truncate)' comment='(20) Truncate' />
+                 </spam>
+             </snf4cgp>
+         </platform>
+
+         <msg-file type='cgp'/>
+
+    </node>
+</snf>
+