Merge remote branch 'origin/topic/appleman/unittests'

Note that I've shifted things around a little bit. The traces are now
in Traces/* (capital T), and the known-hosts-test in policy/*.
Scripts/ was meant just for helper shell scripts for the testing
framework. Also, I've not yet included "policy" in btest.cfg as a
TestDir because the corresponding script is not yet in master.

* origin/topic/appleman/unittests:
  Test, trace file, and baseline for testing the known-services policy script
  Baseline for KNOWN-HOSTS tesT
  Initial drop of known-hosts unit test, with tracefile and directory structure, as well as c ouple of files pulled from Robin's logging branch.

testing/btest/.gitignore

@@ -0,0 +1 @@
+.tmp

testing/btest/Baseline/policy.known-hosts-test/.stderr

@@ -0,0 +1,34 @@
+weird: 1300475167.097012 non_IPv4_packet
+1300475168.652003 weird: bad_TCP_checksum
+1300475168.784020 weird: bad_TCP_checksum
+1300475168.853899 weird: bad_UDP_checksum
+1300475168.854378 weird: bad_UDP_checksum
+1300475168.854837 weird: bad_UDP_checksum
+1300475168.857956 weird: bad_UDP_checksum
+1300475168.858306 weird: bad_UDP_checksum
+1300475168.858713 weird: bad_UDP_checksum
+1300475168.891644 weird: bad_UDP_checksum
+1300475168.892037 weird: bad_UDP_checksum
+1300475168.892414 weird: bad_UDP_checksum
+1300475168.893988 weird: bad_UDP_checksum
+1300475168.894422 weird: bad_UDP_checksum
+1300475168.894787 weird: bad_UDP_checksum
+1300475168.901749 weird: bad_UDP_checksum
+1300475168.902195 weird: bad_UDP_checksum
+1300475168.916018 weird: bad_TCP_checksum
+1300475168.916183 weird: bad_TCP_checksum
+1300475168.918358 weird: bad_TCP_checksum
+1300475168.952296 weird: bad_TCP_checksum
+1300475168.952307 weird: bad_TCP_checksum
+1300475168.954820 weird: bad_TCP_checksum
+1300475168.962687 weird: bad_TCP_checksum
+1300475168.975934 weird: bad_TCP_checksum
+1300475168.976436 weird: bad_TCP_checksum
+1300475168.979264 weird: bad_TCP_checksum
+1300475169.014593 weird: bad_TCP_checksum
+1300475169.014619 weird: bad_TCP_checksum
+1300475169.014927 weird: bad_TCP_checksum
+weird: 1300475171.675372 non_IPv4_packet
+weird: 1300475171.775468 non_IPv4_packet
+weird: 1300475173.116749 non_IPv4_packet
+weird: 1300475173.216550 non_IPv4_packet

testing/btest/Baseline/policy.known-hosts-test/KNOWN_HOSTS

@@ -0,0 +1,5 @@
+ts	address
+1300475168.78384	141.142.220.118	
+1300475168.78384	208.80.152.118	
+1300475168.91594	208.80.152.3	
+1300475168.96263	208.80.152.2	

testing/btest/Baseline/policy.known-services-test/KNOWN_SERVICES

@@ -0,0 +1,229 @@
+ts	host	port_num	service
+964800423.225612	10.20.1.11	80/tcp	{
+
+}	
+964800428.605284	10.20.11.81	3820/tcp	{
+
+}	
+964800432.833579	10.20.11.81	3821/tcp	{
+
+}	
+964800432.220536	10.20.1.8	21/tcp	{
+
+}	
+964800435.180757	10.20.1.8	80/tcp	{
+
+}	
+964800436.657053	10.20.11.81	3822/tcp	{
+
+}	
+964800445.136946	10.20.11.81	3823/tcp	{
+
+}	
+964800447.050657	10.20.11.81	3824/tcp	{
+
+}	
+964800449.128463	10.20.11.81	3825/tcp	{
+
+}	
+964800460.548618	10.20.1.11	220/tcp	{
+
+}	
+964800470.053399	10.20.1.128	32777/tcp	{
+
+}	
+964800470.255065	10.20.1.128	32778/tcp	{
+
+}	
+964800470.326786	10.20.1.128	111/tcp	{
+
+}	
+964800472.135373	10.20.1.128	7/tcp	{
+
+}	
+964800470.551502	10.20.1.128	7100/tcp	{
+
+}	
+964800470.557677	10.20.1.128	32771/tcp	{
+
+}	
+964800470.595967	10.20.1.128	32775/tcp	{
+
+}	
+964800470.695017	10.20.1.128	32779/tcp	{
+
+}	
+964800470.736744	10.20.1.128	32776/tcp	{
+
+}	
+964800470.741478	10.20.1.128	32773/tcp	{
+
+}	
+964800470.735628	10.20.1.128	2049/tcp	{
+
+}	
+964800470.691559	10.20.1.128	6112/tcp	{
+
+}	
+964800470.080672	10.20.1.128	515/tcp	{
+
+}	
+964800470.166217	10.20.1.128	32772/tcp	{
+
+}	
+964800470.023879	10.20.1.128	9/tcp	{
+
+}	
+964800470.248286	10.20.1.128	79/tcp	{
+
+}	
+964800470.173476	10.20.1.128	513/tcp	{
+
+}	
+964800470.288767	10.20.1.128	514/tcp	{
+
+}	
+964800470.688531	10.20.1.128	512/tcp	{
+
+}	
+964800470.211035	10.20.1.128	22/tcp	{
+
+}	
+964800470.49665	10.20.1.128	23/tcp	{
+
+}	
+964800470.066351	10.20.1.128	37/tcp	{
+
+}	
+964800470.599661	10.20.1.128	19/tcp	{
+
+}	
+964800470.552646	10.20.1.128	13/tcp	{
+
+}	
+964800470.557883	10.20.1.128	25/tcp	{
+
+}	
+964800470.784425	10.20.1.128	540/tcp	{
+
+}	
+964800470.292463	10.20.1.128	21/tcp	{
+
+}	
+964800491.909762	10.20.1.133	139/tcp	{
+
+}	
+964800495.488396	10.20.1.32	6000/tcp	{
+
+}	
+964800499.931019	10.20.1.32	22/tcp	{
+
+}	
+964800501.040343	10.20.1.9	80/tcp	{
+
+}	
+964800467.45263	10.20.1.9	110/tcp	{
+
+}	
+964800551.47489	10.20.11.101	79/tcp	{
+
+}	
+964800551.731327	10.20.11.101	22/tcp	{
+
+}	
+964800560.092991	10.20.1.9	22/tcp	{
+
+}	
+964800573.291815	10.20.1.1	22/tcp	{
+
+}	
+964800542.603637	10.20.1.133	21/tcp	{
+
+}	
+964800582.574996	10.20.1.35	25/tcp	{
+
+}	
+964800587.426354	10.20.1.35	21/tcp	{
+
+}	
+964800586.110222	10.20.1.35	23/tcp	{
+
+}	
+964800586.655377	10.20.1.35	22/tcp	{
+
+}	
+964800592.165919	10.20.1.21	21/tcp	{
+
+}	
+964800598.792462	10.20.1.9	25/tcp	{
+
+}	
+964800615.91033	10.20.1.11	21/tcp	{
+
+}	
+964800632.516211	10.20.1.10	25/tcp	{
+
+}	
+964800633.234812	10.20.1.10	80/tcp	{
+
+}	
+964800634.193335	10.20.1.10	21/tcp	{
+
+}	
+964800636.102468	10.20.1.11	443/tcp	{
+
+}	
+964800635.003732	10.20.1.129	79/tcp	{
+
+}	
+964800668.873252	10.20.1.21	110/tcp	{
+
+}	
+964800669.007448	10.20.1.21	111/tcp	{
+
+}	
+964800669.665929	10.20.1.21	80/tcp	{
+
+}	
+964800669.670186	10.20.1.21	690/tcp	{
+
+}	
+964800669.693491	10.20.1.21	2049/tcp	{
+
+}	
+964800670.234186	10.20.1.21	515/tcp	{
+
+}	
+964800670.493675	10.20.1.21	113/tcp	{
+
+}	
+964800670.492704	10.20.1.21	22/tcp	{
+
+}	
+964800676.474475	10.20.1.22	21/tcp	{
+
+}	
+964800673.799597	10.20.1.21	79/tcp	{
+
+}	
+964800675.483229	10.20.1.22	22/tcp	{
+
+}	
+964800670.525453	10.20.1.21	514/tcp	{
+
+}	
+964800703.201113	10.20.1.12	79/tcp	{
+
+}	
+964800633.538485	10.20.11.102	113/tcp	{
+
+}	
+964800470.375479	10.20.1.128	6000/tcp	{
+
+}	
+964800709.128585	10.20.11.102	110/tcp	{
+
+}	
+1184887724.39694	10.20.1.11	2222/tcp	{
+
+}	

testing/btest/Scripts/diff-canonifier

@@ -0,0 +1,5 @@
+#! /usr/bin/env bash
+#
+# Replace anything which looks like timestamps with XXXs.
+
+sed 's/[0-9]\{10\}\.[0-9]\{4,6\}/XXXXXXXXXX.XXXXXX/g'

testing/btest/btest.cfg

@@ -0,0 +1,16 @@
+
+[btest]
+TestDirs    =
+TmpDir      = %(testbase)s/.tmp
+BaselineDir = %(testbase)s/Baseline
+IgnoreDirs  = .svn CVS .tmp
+IgnoreFiles = *.tmp *.swp #*
+
+[environment]
+BROPATH=`bash -c %(testbase)s/../../build/bro-path-dev`
+BRO_SEED_FILE=%(testbase)s/random.seed
+PATH=%(testbase)s/../../build/src:%(testbase)s/../../aux/btest:%(default_path)s
+TEST_DIFF_CANONIFIER=%(testbase)s/Scripts/diff-canonifier
+TRACES=%(testbase)s/Traces
+DIST=%(testbase)s/../..
+BUILD=%(testbase)s/../../build

testing/btest/policy/known-hosts-test

@@ -0,0 +1,6 @@
+
+@TEST-EXEC: bro -r $TRACES/wikipedia.trace known-hosts
+
+@TEST-EXEC: btest-diff KNOWN_HOSTS
+@TEST-EXEC: btest-diff .stderr
+@TEST-EXEC: btest-diff .stdout

testing/btest/policy/known-services-test.bro

@@ -0,0 +1,24 @@
+
+#   Generate some output
+# @TEST-EXEC: bro -r $TRACES/workshop.trace1.trace %INPUT tcp
+
+
+#   Verify the log file, and stderr/out match the Baseline
+# @TEST-EXEC: btest-diff KNOWN_SERVICES
+# @TEST-EXEC: btest-diff .stderr
+# @TEST-EXEC: btest-diff .stdout
+
+
+# Load the script we're here to test
+@load known-services
+
+# Make some changes to how it runs
+export {
+        # Log everything, so we get some output
+        redef KnownServices::logged_hosts=Enabled;
+}
+
+# If necessary, can take setup action here as well
+event bro_init()
+{
+}

testing/btest/random.seed

@@ -0,0 +1,17 @@
+2983378351
+1299727368
+0
+310447
+0
+1409073626
+3975311262
+34130240
+1450515018
+1466150520
+1342286698
+1193956778
+2188527278
+3361989254
+3912865238
+3596260151
+517973768