Merge remote branch 'remotes/origin/topic/policy-scripts-new'

* remotes/origin/topic/policy-scripts-new: (264 commits)
  Update core.vlan-mpls baseline.
  Update core.conn-uid test/baseline.
  Fixing tests / updating baselines.
  Renaming a poorly named notice.
  Resolving a conflict in scan.l.
  Notice framework updates.
  Fixed bug I just introduced to known-services.
  Known services now only logs for "normal" connections.
  Fix the name of the DPD log.
  Updating the netstats script to match the new internal code.
  Removed expected_connection_timeout variable.
  New default notice actions for emailing network admins.
  New default notice action for emailing network admins.
  weird.bro moved into notice framework.
  Removed a redundant and out of place mozilla CA list.
  Renamed the stream for the notice log.
  Small consistency tweaks for the communications framework.
  Finishing the inactivity script.
  Intel framework test passes now.
  Reorganized some of the tests for consistency.
  ...

bro-path-dev.in

@@ -10,12 +10,12 @@
 #     BROPATH=`./bro-path-dev` ./src/bro
 #
 
-broPolicies=${PROJECT_SOURCE_DIR}/policy:${PROJECT_SOURCE_DIR}/policy/sigs:${PROJECT_SOURCE_DIR}/policy/time-machine
+broPolicies=${PROJECT_SOURCE_DIR}/policy:${PROJECT_SOURCE_DIR}/policy/frameworks:${PROJECT_SOURCE_DIR}/policy/protocols:${PROJECT_SOURCE_DIR}/policy/detectors:${PROJECT_SOURCE_DIR}/policy/time-machine
 
 broGenPolicies=${CMAKE_BINARY_DIR}/src
 
 broctlPolicies=${PROJECT_SOURCE_DIR}/aux/broctl/policy:${CMAKE_BINARY_DIR}/aux/broctl/policy/local
 
-installedPolicies=${POLICYDIR}:${POLICYDIR}/sigs:${POLICYDIR}/time-machine:${POLICYDIR}/site
+installedPolicies=${POLICYDIR}:${POLICYDIR}/frameworks:${POLICYDIR}/protocols:${POLICYDIR}/detectors:${POLICYDIR}/time-machine:${POLICYDIR}/site
 
 echo .:$broPolicies:$broGenPolicies:$broctlPolicies

doc/scripts/CMakeLists.txt

@@ -59,8 +59,6 @@ macro(REST_TARGET srcDir broInput)
         set(basename "${basename}.init")
     endif ()
 
-    set (restFile "${basename}.rst")
-
     if (NOT relDstDir)
         set(docName "${basename}")
         set(dstDir "${RST_OUTPUT_DIR}")
@@ -69,7 +67,9 @@ macro(REST_TARGET srcDir broInput)
         set(dstDir "${RST_OUTPUT_DIR}/${relDstDir}")
     endif ()
 
-    set(restOutput "${dstDir}/${restFile}")
+    set(restFile "${docName}.rst")
+    string(REPLACE "/" "^" restFile ${restFile})
+    set(restOutput "${dstDir}/${basename}.rst")
 
     set(indexEntry "   ${docName} <${docName}>")
     set(MASTER_POLICY_INDEX_TEXT "${MASTER_POLICY_INDEX_TEXT}\n${indexEntry}")
@@ -97,7 +97,7 @@ macro(REST_TARGET srcDir broInput)
     if (${group} STREQUAL "default" OR ${group} STREQUAL "bifs")
         set(BRO_ARGS --doc-scripts --exec '')
     else ()
-        set(BRO_ARGS --doc-scripts ${srcDir}/${broInput})
+        set(BRO_ARGS --doc-scripts ${broInput})
     endif ()
 
     add_custom_command(OUTPUT ${restOutput}
@@ -105,7 +105,7 @@ macro(REST_TARGET srcDir broInput)
         COMMAND "${CMAKE_COMMAND}"
         ARGS -E remove_directory .state
         # generate the reST documentation using bro
-        COMMAND BROPATH=${BROPATH} ${CMAKE_BINARY_DIR}/src/bro
+        COMMAND BROPATH=${BROPATH}:${srcDir} ${CMAKE_BINARY_DIR}/src/bro
         ARGS ${BRO_ARGS} || (rm -rf .state *.log *.rst && exit 1)
         # move generated doc into a new directory tree that
         # defines the final structure of documents
@@ -129,26 +129,102 @@ endmacro(REST_TARGET)
 # Schedule Bro scripts for which to generate documentation.
 # Note: the script may be located in a subdirectory off of one of the main
 # directories in BROPATH.  In that case, just list the script as 'foo/bar.bro'
-rest_target(${POLICY_SRC_DIR} alarm.bro                         user)
-rest_target(${POLICY_SRC_DIR} arp.bro                           user)
-rest_target(${POLICY_SRC_DIR} conn.bro                          user)
-rest_target(${POLICY_SRC_DIR} dhcp.bro                          user)
-rest_target(${POLICY_SRC_DIR} dns.bro                           user)
-rest_target(${POLICY_SRC_DIR} ftp.bro                           user)
-rest_target(${POLICY_SRC_DIR} http.bro                          user)
-rest_target(${POLICY_SRC_DIR} http-reply.bro                    user)
-rest_target(${POLICY_SRC_DIR} http-request.bro                  user)
-rest_target(${POLICY_SRC_DIR} irc.bro                           user)
-rest_target(${POLICY_SRC_DIR} smtp.bro                          user)
-rest_target(${POLICY_SRC_DIR} ssl.bro                           user)
-rest_target(${POLICY_SRC_DIR} ssl-ciphers.bro                   user)
-rest_target(${POLICY_SRC_DIR} ssl-errors.bro                    user)
-rest_target(${POLICY_SRC_DIR} synflood.bro                      user)
-rest_target(${POLICY_SRC_DIR} tcp.bro                           user)
-rest_target(${POLICY_SRC_DIR} udp.bro                           user)
-rest_target(${POLICY_SRC_DIR} weird.bro                         user)
 rest_target(${CMAKE_CURRENT_SOURCE_DIR} example.bro           internal)
 
+rest_target(${POLICY_SRC_DIR} conn.bro                        user)
+rest_target(${POLICY_SRC_DIR} conn/base.bro                   user)
+rest_target(${POLICY_SRC_DIR} conn/contents.bro               user)
+
+rest_target(${POLICY_SRC_DIR} dns.bro                         policy/dns-index)
+rest_target(${POLICY_SRC_DIR} dns/auth-addl.bro               policy/dns-index)
+rest_target(${POLICY_SRC_DIR} dns/base.bro                    policy/dns-index)
+rest_target(${POLICY_SRC_DIR} dns/consts.bro                  policy/dns-index)
+rest_target(${POLICY_SRC_DIR} dns/detect.bro                  policy/dns-index)
+rest_target(${POLICY_SRC_DIR} dns/passive-replication.bro     policy/dns-index)
+
+# TODO: these don't currently work due to something that looks like a
+# circular dependency.  They'll also change to the 'default' group once
+# loaded from bro.init.
+#rest_target(${POLICY_SRC_DIR} dpd.bro                         policy/dpd-index)
+#rest_target(${POLICY_SRC_DIR} dpd/base.bro                    policy/dpd-index)
+#rest_target(${POLICY_SRC_DIR} dpd/dyn-disable.bro             policy/dpd-index)
+#rest_target(${POLICY_SRC_DIR} dpd/packet-segment-logging.bro  policy/dpd-index)
+
+rest_target(${POLICY_SRC_DIR} ftp.bro                         policy/ftp-index)
+rest_target(${POLICY_SRC_DIR} ftp/base.bro                    policy/ftp-index)
+rest_target(${POLICY_SRC_DIR} ftp/detect.bro                  policy/ftp-index)
+rest_target(${POLICY_SRC_DIR} ftp/file-extract.bro            policy/ftp-index)
+rest_target(${POLICY_SRC_DIR} ftp/software.bro                policy/ftp-index)
+rest_target(${POLICY_SRC_DIR} ftp/utils-commands.bro          policy/ftp-index)
+
+rest_target(${POLICY_SRC_DIR} functions.bro                   user)
+
+# TODO: hot.conn.bro currently won't load because hot.bro doesn't exist
+#rest_target(${POLICY_SRC_DIR} hot.conn.bro                    user)
+
+rest_target(${POLICY_SRC_DIR} http.bro                        policy/http-index)
+rest_target(${POLICY_SRC_DIR} http/base-extended.bro          policy/http-index)
+rest_target(${POLICY_SRC_DIR} http/base.bro                   policy/http-index)
+rest_target(${POLICY_SRC_DIR} http/detect-intel.bro           policy/http-index)
+rest_target(${POLICY_SRC_DIR} http/detect-sqli.bro            policy/http-index)
+rest_target(${POLICY_SRC_DIR} http/detect-webapps.bro         policy/http-index)
+rest_target(${POLICY_SRC_DIR} http/file-extract.bro           policy/http-index)
+rest_target(${POLICY_SRC_DIR} http/file-hash.bro              policy/http-index)
+rest_target(${POLICY_SRC_DIR} http/file-ident.bro             policy/http-index)
+rest_target(${POLICY_SRC_DIR} http/headers.bro                policy/http-index)
+rest_target(${POLICY_SRC_DIR} http/software.bro               policy/http-index)
+rest_target(${POLICY_SRC_DIR} http/utils.bro                  policy/http-index)
+rest_target(${POLICY_SRC_DIR} http/var-extraction-cookies.bro policy/http-index)
+rest_target(${POLICY_SRC_DIR} http/var-extraction-uri.bro     policy/http-index)
+
+rest_target(${POLICY_SRC_DIR} irc.bro                         policy/irc-index)
+rest_target(${POLICY_SRC_DIR} irc/base.bro                    policy/irc-index)
+rest_target(${POLICY_SRC_DIR} irc/dcc-send.bro                policy/irc-index)
+
+rest_target(${POLICY_SRC_DIR} known-services.bro              user)
+rest_target(${POLICY_SRC_DIR} known-hosts.bro                 user)
+
+rest_target(${POLICY_SRC_DIR} metrics.bro                  policy/metrics-index)
+rest_target(${POLICY_SRC_DIR} metrics/base.bro             policy/metrics-index)
+rest_target(${POLICY_SRC_DIR} metrics/conn-example.bro     policy/metrics-index)
+rest_target(${POLICY_SRC_DIR} metrics/http-example.bro     policy/metrics-index)
+
+rest_target(${POLICY_SRC_DIR} mime.bro                        policy/mime-index)
+rest_target(${POLICY_SRC_DIR} mime/base.bro                   policy/mime-index)
+rest_target(${POLICY_SRC_DIR} mime/file-extract.bro           policy/mime-index)
+rest_target(${POLICY_SRC_DIR} mime/file-hash.bro              policy/mime-index)
+rest_target(${POLICY_SRC_DIR} mime/file-ident.bro             policy/mime-index)
+
+rest_target(${POLICY_SRC_DIR} notice-action-filters.bro       user)
+rest_target(${POLICY_SRC_DIR} notice.bro                      user)
+rest_target(${POLICY_SRC_DIR} site.bro                        user)
+
+rest_target(${POLICY_SRC_DIR} signatures.bro                  policy/sig-index)
+rest_target(${POLICY_SRC_DIR} signatures/base.bro             policy/sig-index)
+
+rest_target(${POLICY_SRC_DIR} smtp.bro                        policy/smtp-index)
+rest_target(${POLICY_SRC_DIR} smtp/base-extended.bro          policy/smtp-index)
+rest_target(${POLICY_SRC_DIR} smtp/base.bro                   policy/smtp-index)
+rest_target(${POLICY_SRC_DIR} smtp/detect.bro                 policy/smtp-index)
+rest_target(${POLICY_SRC_DIR} smtp/software.bro               policy/smtp-index)
+rest_target(${POLICY_SRC_DIR} smtp/utils.bro                  policy/smtp-index)
+rest_target(${POLICY_SRC_DIR} smtp/webmail-ident.bro          policy/smtp-index)
+
+rest_target(${POLICY_SRC_DIR} software.bro                policy/software-index)
+rest_target(${POLICY_SRC_DIR} software/base.bro           policy/software-index)
+rest_target(${POLICY_SRC_DIR} software/vulnerable.bro     policy/software-index)
+
+rest_target(${POLICY_SRC_DIR} ssh.bro                         policy/ssh-index)
+rest_target(${POLICY_SRC_DIR} ssh/base.bro                    policy/ssh-index)
+rest_target(${POLICY_SRC_DIR} ssh/software.bro                policy/ssh-index)
+
+rest_target(${POLICY_SRC_DIR} ssl-ciphers.bro                 policy/ssl-index)
+rest_target(${POLICY_SRC_DIR} ssl-errors.bro                  policy/ssl-index)
+rest_target(${POLICY_SRC_DIR} ssl.bro                         policy/ssl-index)
+
+rest_target(${POLICY_SRC_DIR} utils/pattern.bro               user)
+rest_target(${POLICY_SRC_DIR} weird.bro                       user)
+
 # Finding out what scripts bro will generate documentation for by default
 # can be done like: `bro --doc-scripts --exec ""`
 rest_target(${POLICY_SRC_DIR} bro.init                         default)

doc/scripts/example.bro

@@ -66,7 +66,7 @@ redef dpd_config += {
 
 # redefinitions of "Notice::Type" are self-documenting, but
 # more information can be supplied in two different ways
-redef enum Notice += {
+redef enum Notice::Type += {
     ## any number of this type of comment
     ## will document "Notice_One"
     Notice_One,

doc/scripts/source/bifs.rst

@@ -1,4 +1,5 @@
+.. This is a stub doc to which the build process can append.
+
 Built-In Functions (BIFs)
 =========================
 
-Here's a list of all documentation for BIFs that Bro provides:

doc/scripts/source/index.rst

@@ -11,11 +11,22 @@ Contents:
 
    common
    builtins
-   policy/index
    default
    bifs
    user
+   policy/dns-index
+   policy/ftp-index
+   policy/http-index
+   policy/irc-index
+   policy/metrics-index
+   policy/mime-index
+   policy/sig-index
+   policy/smtp-index
+   policy/software-index
+   policy/ssh-index
+   policy/ssl-index
    internal
+   policy/index
 
 Indices and tables
 ==================

doc/scripts/source/internal.rst

@@ -1,3 +1,5 @@
+.. This is a stub doc to which the build process can append.
+
 Internal Policy Scripts
 =======================
 

doc/scripts/source/policy/dns-index.rst

@@ -0,0 +1,5 @@
+.. This is a stub doc to which the build process can append.
+
+DNS Policy Scripts
+==================
+

doc/scripts/source/policy/ftp-index.rst

@@ -0,0 +1,5 @@
+.. This is a stub doc to which the build process can append.
+
+FTP Policy Scripts
+==================
+

doc/scripts/source/policy/http-index.rst

@@ -0,0 +1,5 @@
+.. This is a stub doc to which the build process can append.
+
+HTTP Policy Scripts
+===================
+

doc/scripts/source/policy/irc-index.rst

@@ -0,0 +1,5 @@
+.. This is a stub doc to which the build process can append.
+
+IRC Policy Scripts
+==================
+

doc/scripts/source/policy/metrics-index.rst

@@ -0,0 +1,5 @@
+.. This is a stub doc to which the build process can append.
+
+Metrics Policy Scripts
+======================
+

doc/scripts/source/policy/mime-index.rst

@@ -0,0 +1,5 @@
+.. This is a stub doc to which the build process can append.
+
+MIME Policy Scripts
+===================
+

doc/scripts/source/policy/sig-index.rst

@@ -0,0 +1,5 @@
+.. This is a stub doc to which the build process can append.
+
+Signature Policy Scripts
+========================
+

doc/scripts/source/policy/smtp-index.rst

@@ -0,0 +1,5 @@
+.. This is a stub doc to which the build process can append.
+
+SMTP Policy Scripts
+===================
+

doc/scripts/source/policy/software-index.rst

@@ -0,0 +1,5 @@
+.. This is a stub doc to which the build process can append.
+
+Software Policy Scripts
+=======================
+

doc/scripts/source/policy/ssh-index.rst

@@ -0,0 +1,5 @@
+.. This is a stub doc to which the build process can append.
+
+SSH Policy Scripts
+==================
+

doc/scripts/source/policy/ssl-index.rst

@@ -0,0 +1,5 @@
+.. This is a stub doc to which the build process can append.
+
+SSL Policy Scripts
+==================
+

doc/scripts/source/user.rst

@@ -1,3 +1,5 @@
-User-Facing Policy Scripts
+.. This is a stub doc to which the build process can append.
-==========================
+
+Other User-Facing Policy Scripts
+================================
 

policy.old/all.bro

@@ -0,0 +1,141 @@
+@load heavy-analysis
+@load OS-fingerprint
+@load adu
+@load alarm
+@load analy
+@load anon
+@load arp
+@load backdoor
+@load bittorrent
+@load blaster
+@load bt-tracker
+@load brolite-backdoor
+@load capture-events
+@load capture-loss
+@load capture-state-updates
+@load checkpoint
+@load clear-passwords
+@load conn-flood
+@load conn-id
+@load conn
+@load contents
+@load cpu-adapt
+@load dce
+@load demux
+@load detect-protocols-http
+@load detect-protocols
+@load dhcp
+@load dns-info
+@load dns-lookup
+@load dns
+@load dpd
+@load drop-adapt
+@load dyn-disable
+@load file-flush
+@load finger
+@load firewall
+@load flag-irc
+@load flag-warez
+@load frag
+@load ftp
+@load gnutella
+@load hot-ids
+@load hot
+@load http-abstract
+@load http-anon-server
+@load http-anon-useragent
+@load http-anon-utils
+@load http-body
+@load http-detect-passwd
+@load http-entity
+@load http-event
+@load http-header
+@load http-identified-files.bro
+@load http-reply
+@load http-request
+@load http-rewriter
+@load http
+@load icmp
+@load ident-rewriter
+@load ident
+@load inactivity
+@load interconn
+@load irc-bot-syslog
+@load irc-bot
+@load irc
+@load large-conns
+@load listen-clear
+@load listen-ssl
+@load load-level
+@load load-sample
+@load log-append
+@load login
+@load mime-pop
+@load mime
+@load mt
+@load ncp
+@load netflow
+@load netstats
+@load nfs
+@load notice-action-filters
+@load notice
+@load ntp
+@load passwords
+@load pcap
+@load pkt-profile
+@load pop3
+@load port-name
+@load portmapper
+@load print-filter
+@load print-globals
+@load print-resources
+@load print-sig-states
+@load profiling
+@load proxy
+@load remote-pcap
+@load remote-ping
+@load remote-print-id-reply
+@load remote-print-id
+@load remote-print
+@load remote-report-notices
+@load remote-send-id
+@load remote
+@load rotate-logs
+@load rsh
+@load scan
+@load secondary-filter
+@load sensor-sshd
+@load server-ports
+@load service-probe
+@load signatures
+@load site
+@load smb
+@load smtp-relay
+@load smtp-rewriter
+@load smtp
+@load snort
+@load software
+@load ssh
+@load ssh-stepping
+@load ssl-alerts
+@load ssl-ciphers
+@load ssl-errors
+@load ssl-worm
+@load ssl
+@load stats
+@load stepping
+@load synflood
+@load targeted-scan
+@load tcp
+@load tftp
+@load trw-impl
+@load trw
+@load udp-common
+@load udp
+@load vlan
+@load weird
+@load worm
+@load notice-policy
+
+# The following keeps us running after the bro_init event.
+redef PrintFilter::terminate_bro = F;

policy.old/conn.bro

@@ -15,22 +15,13 @@ const conn_closed = { TCP_CLOSED, TCP_RESET };
 global have_FTP = F;	# if true, we've loaded ftp.bro
 global have_SMTP = F;	# if true, we've loaded smtp.bro
 
-# TODO: Do we have a nicer way of defining this prototype?
+# TODO: Do we have a nicer way of doing this?
 export { global FTP::is_ftp_data_conn: function(c: connection): bool; }
 
 # Whether to include connection state history in the logs generated
 # by record_connection.
 const record_state_history = F &redef;
 
-# Whether to add 4 more columns to conn.log with 
-# orig_packet orig_ip_bytes resp_packets resp_ip_bytes
-# Requires use_conn_size_analyzer=T
-# Columns are added after history but before addl
-const report_conn_size_analyzer = F &redef;
-
-# Activate conn-size analyzer if necessary.
-redef use_conn_size_analyzer = (! report_conn_size_analyzer);
-
 # Whether to translate the local address in SensitiveConnection notices
 # to a hostname.  Meant as a demonstration of the "when" construct.
 const xlate_hot_local_addr = F &redef;
@@ -105,12 +96,6 @@ function conn_size(e: endpoint, trans: transport_proto): string
 		return "?";
 	}
 
-function conn_size_from_analyzer(e: endpoint): string
-	{
-	return fmt("%d %d", (e?$num_pkts) ? e$num_pkts : 0,
-			(e?$num_bytes_ip) ? e$num_bytes_ip : 0);
-	}
-
 function service_name(c: connection): string
 	{
 	local p = c$id$resp_p;
@@ -319,35 +304,12 @@ function record_connection(f: file, c: connection)
 		log_msg = fmt("%s %s", log_msg,
 				c$history == "" ? "X" : c$history);
 
-	if ( use_conn_size_analyzer && report_conn_size_analyzer )
-		log_msg = fmt("%s %s %s", log_msg, 
-				conn_size_from_analyzer(c$orig),  conn_size_from_analyzer(c$resp));
-
 	if ( addl != "" )
 		log_msg = fmt("%s %s", log_msg, addl);
 
 	print f, log_msg;
 	}
 
-event protocol_confirmation(c: connection, atype: count, aid: count)
-	{
-	if ( ! dpd_conn_logs )
-		return;
-
-	delete c$service[fmt("-%s",analyzer_name(atype))];
-	add c$service[analyzer_name(atype)];
-	}
-
-event protocol_violation(c: connection, atype: count, aid: count,
-				reason: string) &priority = 10
-	{
-	if ( ! dpd_conn_logs )
-		return;
-
-	delete c$service[analyzer_name(atype)];
-	add c$service[fmt("-%s",analyzer_name(atype))];
-	}
-
 event connection_established(c: connection)
 	{
 	Hot::check_hot(c, Hot::CONN_ESTABLISHED);