Add Conn and DNS protocol script documentation. (fixes #731)

scripts/base/protocols/conn/contents.bro

@@ -1,23 +1,27 @@
 ##! This script can be used to extract either the originator's data or the 
 ##! responders data or both.  By default nothing is extracted, and in order 
 ##! to actually extract data the ``c$extract_orig`` and/or the
-##! ``c$extract_resp`` variable must be set to T.  One way to achieve this
+##! ``c$extract_resp`` variable must be set to ``T``.  One way to achieve this
-##! would be to handle the connection_established event elsewhere and set the
+##! would be to handle the :bro:id:`connection_established` event elsewhere
-##! extract_orig and extract_resp options there.  However, there may be trouble
+##! and set the ``extract_orig`` and ``extract_resp`` options there.
-##! with the timing due the event queue delay.
+##! However, there may be trouble with the timing due to event queue delay.
-##! This script does not work well in a cluster context unless it has a 
+##!
-##! remotely mounted disk to write the content files to.
+##! .. note::
+##!
+##!    This script does not work well in a cluster context unless it has a
+##!    remotely mounted disk to write the content files to.
 
 @load base/utils/files
 
 module Conn;
 
 export {
-	## The prefix given to files as they are opened on disk.
+	## The prefix given to files containing extracted connections as they are
+	## opened on disk.
 	const extraction_prefix = "contents" &redef;
 	
-	## If this variable is set to T, then all contents of all files will be 
+	## If this variable is set to ``T``, then all contents of all connections
-	## extracted.
+	## will be  extracted.
 	const default_extract = F &redef;
 }
 

scripts/base/protocols/conn/inactivity.bro

@@ -4,7 +4,7 @@
 module Conn;
 
 export {
-	## Define inactivty timeouts by the service detected being used over
+	## Define inactivity timeouts by the service detected being used over
 	## the connection.
 	const analyzer_inactivity_timeouts: table[AnalyzerTag] of interval = {
 		# For interactive services, allow longer periods of inactivity.

scripts/base/protocols/conn/main.bro

@@ -1,17 +1,33 @@
+##! This script manages the tracking/logging of general information regarding
+##! TCP, UDP, and ICMP traffic.  For UDP and ICMP, "connections" are to
+##! be interpreted using flow semantics (sequence of packets from a source
+##! host/post to a destination host/port).  Further, ICMP "ports" are to
+##! be interpreted as the source port meaning the ICMP message type and
+##! the destination port being the ICMP message code.
+
 @load base/utils/site
 
 module Conn;
 
 export {
+	## The connection logging stream identifier.
 	redef enum Log::ID += { LOG };
 
+	## The record type which contains column fields of the connection log.
 	type Info: record {
 		## This is the time of the first packet.
 		ts:           time            &log;
+		## A unique identifier of a connection.
 		uid:          string          &log;
+		## The connection's 4-tuple of endpoint addresses/ports.
 		id:           conn_id         &log;
+		## The transport layer protocol of the connection.
 		proto:        transport_proto &log;
+		## An identification of an application protocol being sent over the
+		## the connection.
 		service:      string          &log &optional;
+		## How long the connection lasted.  For 3-way or 4-way connection
+		## tear-downs, this will not include the final ACK.
 		duration:     interval        &log &optional;
 		## The number of payload bytes the originator sent. For TCP
 		## this is taken from sequence numbers and might be inaccurate 
@@ -51,8 +67,8 @@ export {
 		## have been completed prior to the packet loss.
 		missed_bytes: count           &log &default=0;
 
-		## Records the state history of (TCP) connections as
+		## Records the state history of connections as a string of letters.
-		## a string of letters.
+		## For TCP connections the meaning of those letters is:
 		##
 		## ======  ====================================================
 		## Letter  Meaning
@@ -71,7 +87,8 @@ export {
 		## originator and lower case then means the responder.
 		## Also, there is compression. We only record one "d" in each direction,
 		## for instance. I.e., we just record that data went in that direction.
-		## This history is not meant to encode how much data that happened to be.
+		## This history is not meant to encode how much data that happened to
+		## be.
 		history:      string          &log &optional;
 		## Number of packets the originator sent.
 		## Only set if :bro:id:`use_conn_size_analyzer` = T
@@ -86,6 +103,8 @@ export {
 		resp_ip_bytes: count      &log &optional;
 	};
 
+	## Event that can be handled to access the :bro:type:`Conn::Info`
+	## record as it is sent on to the logging framework.
 	global log_conn: event(rec: Info);
 }
 

scripts/base/protocols/dns/consts.bro

@@ -4,9 +4,9 @@
 module DNS;
 
 export {
-	const PTR = 12;
+	const PTR = 12;  ##< RR TYPE value for a domain name pointer.
-	const EDNS = 41;
+	const EDNS = 41; ##< An OPT RR TYPE value described by EDNS.
-	const ANY = 255;
+	const ANY = 255; ##< A QTYPE value describing a request for all records.
 
 	## Mapping of DNS query type codes to human readable string representation.
 	const query_types = {
@@ -29,50 +29,43 @@ export {
 		[ANY] = "*",
 	} &default = function(n: count): string { return fmt("query-%d", n); };
 
-	const code_types = {
-		[0] = "X0",
-		[1] = "Xfmt",
-		[2] = "Xsrv",
-		[3] = "Xnam",
-		[4] = "Ximp",
-		[5] = "X[",
-	} &default="?";
-
 	## Errors used for non-TSIG/EDNS types.
 	const base_errors = {
-		[0] = "NOERROR",        ##< No Error
+		[0] = "NOERROR",        # No Error
-		[1] = "FORMERR",        ##< Format Error
+		[1] = "FORMERR",        # Format Error
-		[2] = "SERVFAIL",       ##< Server Failure
+		[2] = "SERVFAIL",       # Server Failure
-		[3] = "NXDOMAIN",       ##< Non-Existent Domain
+		[3] = "NXDOMAIN",       # Non-Existent Domain
-		[4] = "NOTIMP",         ##< Not Implemented
+		[4] = "NOTIMP",         # Not Implemented
-		[5] = "REFUSED",        ##< Query Refused
+		[5] = "REFUSED",        # Query Refused
-		[6] = "YXDOMAIN",       ##< Name Exists when it should not
+		[6] = "YXDOMAIN",       # Name Exists when it should not
-		[7] = "YXRRSET",        ##< RR Set Exists when it should not
+		[7] = "YXRRSET",        # RR Set Exists when it should not
-		[8] = "NXRRSet",        ##< RR Set that should exist does not
+		[8] = "NXRRSet",        # RR Set that should exist does not
-		[9] = "NOTAUTH",        ##< Server Not Authoritative for zone
+		[9] = "NOTAUTH",        # Server Not Authoritative for zone
-		[10] = "NOTZONE",       ##< Name not contained in zone
+		[10] = "NOTZONE",       # Name not contained in zone
-		[11] = "unassigned-11", ##< available for assignment
+		[11] = "unassigned-11", # available for assignment
-		[12] = "unassigned-12", ##< available for assignment
+		[12] = "unassigned-12", # available for assignment
-		[13] = "unassigned-13", ##< available for assignment
+		[13] = "unassigned-13", # available for assignment
-		[14] = "unassigned-14", ##< available for assignment
+		[14] = "unassigned-14", # available for assignment
-		[15] = "unassigned-15", ##< available for assignment
+		[15] = "unassigned-15", # available for assignment
-		[16] = "BADVERS",       ##< for EDNS, collision w/ TSIG
+		[16] = "BADVERS",       # for EDNS, collision w/ TSIG
-		[17] = "BADKEY",        ##< Key not recognized
+		[17] = "BADKEY",        # Key not recognized
-		[18] = "BADTIME",       ##< Signature out of time window
+		[18] = "BADTIME",       # Signature out of time window
-		[19] = "BADMODE",       ##< Bad TKEY Mode
+		[19] = "BADMODE",       # Bad TKEY Mode
-		[20] = "BADNAME",       ##< Duplicate key name
+		[20] = "BADNAME",       # Duplicate key name
-		[21] = "BADALG",        ##< Algorithm not supported
+		[21] = "BADALG",        # Algorithm not supported
-		[22] = "BADTRUNC",      ##< draft-ietf-dnsext-tsig-sha-05.txt
+		[22] = "BADTRUNC",      # draft-ietf-dnsext-tsig-sha-05.txt
-		[3842] = "BADSIG",      ##< 16 <= number collision with EDNS(16);
+		[3842] = "BADSIG",      # 16 <= number collision with EDNS(16);
-		                        ##< this is a translation from TSIG(16)
+		                        # this is a translation from TSIG(16)
 	} &default = function(n: count): string { return fmt("rcode-%d", n); };
 
-	# This deciphers EDNS Z field values.
+	## This deciphers EDNS Z field values.
 	const edns_zfield = {
 		[0]     = "NOVALUE",    # regular entry
 		[32768] = "DNS_SEC_OK", # accepts DNS Sec RRs
 	} &default="?";
 
+	## Possible values of the CLASS field in resource records or QCLASS field
+	## in query messages.
 	const classes = {
 		[1]   = "C_INTERNET",
 		[2]   = "C_CSNET",

scripts/base/protocols/dns/main.bro

@@ -1,38 +1,80 @@
+##! Base DNS analysis script which tracks and logs DNS queries along with
+##! their responses.
+
 @load ./consts
 
 module DNS;
 
 export {
+	## The DNS logging stream identifier.
 	redef enum Log::ID += { LOG };
 
+	## The record type which contains the column fields of the DNS log.
 	type Info: record {
+		## The earliest time at which a DNS protocol message over the
+		## associated connection is observed.
 		ts:            time               &log;
+		## A unique identifier of the connection over which DNS messages
+		## are being transferred.
 		uid:           string             &log;
+		## The connection's 4-tuple of endpoint addresses/ports.
 		id:            conn_id            &log;
+		## The transport layer protocol of the connection.
 		proto:         transport_proto    &log;
+		## A 16 bit identifier assigned by the program that generated the
+		## DNS query.  Also used in responses to match up replies to
+		## outstanding queries.
 		trans_id:      count              &log &optional;
+		## The domain name that is the subject of the DNS query.
 		query:         string             &log &optional;
+		## The QCLASS value specifying the class of the query.
 		qclass:        count              &log &optional;
+		## A descriptive name for the class of the query.
 		qclass_name:   string             &log &optional;
+		## A QTYPE value specifying the type of the query.
 		qtype:         count              &log &optional;
+		## A descriptive name for the type of the query.
 		qtype_name:    string             &log &optional;
+		## The response code value in DNS response messages.
 		rcode:         count              &log &optional;
+		## A descriptive name for the response code value.
 		rcode_name:    string             &log &optional;
+		## Whether the message is a query (F) or response (T).
 		QR:            bool               &log &default=F;
+		## The Authoritative Answer bit for response messages specifies that
+		## the responding name server is an authority for the domain name
+		## in the question section.
 		AA:            bool               &log &default=F;
+		## The Truncation bit specifies that the message was truncated.
 		TC:            bool               &log &default=F;
+		## The Recursion Desired bit indicates to a name server to recursively
+		## purse the query.
 		RD:            bool               &log &default=F;
+		## The Recursion Available bit in a response message indicates if
+		## the name server supports recursive queries.
 		RA:            bool               &log &default=F;
+		## A reserved field that is currently supposed to be zero in all
+		## queries and responses.
 		Z:             count              &log &default=0;
+		## The set of resource descriptions in answer of the query.
 		answers:       vector of string   &log &optional;
+		## The caching intervals of the associated RRs described by the
+		## ``answers`` field.
 		TTLs:          vector of interval &log &optional;
 
-		## This value indicates if this request/response pair is ready to be logged.
+		## This value indicates if this request/response pair is ready to be
+		## logged.
 		ready:         bool            &default=F;
+		## The total number of resource records in a reply message's answer
+		## section.
 		total_answers: count           &optional;
+		## The total number of resource records in a reply message's answer,
+		## authority, and additional sections.
 		total_replies: count           &optional;
 	};
 
+	## A record type which tracks the status of DNS queries for a given
+	## :bro:type:`connection`.
 	type State: record {
 		## Indexed by query id, returns Info record corresponding to
 		## query/response which haven't completed yet.
@@ -44,11 +86,21 @@ export {
 		finished_answers: set[count] &optional;
 	};
 
+	## An event that can be handled to access the :bro:type:`DNS::Info`
+	## record as it is sent to the logging framework.
 	global log_dns: event(rec: Info);
 
 	## This is called by the specific dns_*_reply events with a "reply" which
 	## may not represent the full data available from the resource record, but
 	## it's generally considered a summarization of the response(s).
+	##
+	## c: The connection record for which to fill in DNS reply data.
+	##
+	## msg: The DNS message header information for the response.
+	##
+	## ans: The general information of a RR response.
+	##
+	## reply: The specific response information according to RR type/class.
 	global do_reply: event(c: connection, msg: dns_msg, ans: dns_answer, reply: string);
 }
 

scripts/policy/protocols/conn/known-hosts.bro

@@ -8,8 +8,10 @@
 module Known;
 
 export {
+	## The known-hosts logging stream identifier.
 	redef enum Log::ID += { HOSTS_LOG };
 
+	## The record type which contains the column fields of the known-hosts log.
 	type HostsInfo: record {
 		## The timestamp at which the host was detected.
 		ts:      time &log;
@@ -19,7 +21,7 @@ export {
 	};
 	
 	## The hosts whose existence should be logged and tracked.
-	## Choices are: LOCAL_HOSTS, REMOTE_HOSTS, ALL_HOSTS, NO_HOSTS
+	## See :bro:type:`Host` for possible choices.
 	const host_tracking = LOCAL_HOSTS &redef;
 	
 	## The set of all known addresses to store for preventing duplicate 
@@ -29,6 +31,8 @@ export {
 	## of each individual address is logged each day.
 	global known_hosts: set[addr] &create_expire=1day &synchronized &redef;
 
+	## An event that can be handled to access the :bro:type:`Known::HostsInfo`
+	## record as it is sent on to the logging framework.
 	global log_known_hosts: event(rec: HostsInfo);
 }
 

scripts/policy/protocols/conn/known-services.bro

@@ -8,29 +8,41 @@
 module Known;
 
 export {
+	## The known-services logging stream identifier.
 	redef enum Log::ID += { SERVICES_LOG };
 
+	## The record type which contains the column fields of the known-services
+	## log.
 	type ServicesInfo: record {
+		## The time at which the service was detected.
 		ts:             time            &log;
+		## The host address on which the service is running.
 		host:           addr            &log;
+		## The port number on which the service is running.
 		port_num:       port            &log;
+		## The transport-layer protocol which the service uses.
 		port_proto:     transport_proto &log;
+		## A set of protocols that match the service's connection payloads.
 		service:        set[string]     &log;
-		
-		done:           bool &default=F;
 	};
 	
 	## The hosts whose services should be tracked and logged.
+	## See :bro:type:`Host` for possible choices.
 	const service_tracking = LOCAL_HOSTS &redef;
 
+	## Tracks the set of daily-detected services for preventing the logging
+	## of duplicates, but can also be inspected by other scripts for
+	## different purposes.
 	global known_services: set[addr, port] &create_expire=1day &synchronized;
 
+	## Event that can be handled to access the :bro:type:`Known::ServicesInfo`
+	## record as it is sent on to the logging framework.
 	global log_known_services: event(rec: ServicesInfo);
 }
 
 redef record connection += {
-	## This field is to indicate whether or not the processing for detecting 
+	# This field is to indicate whether or not the processing for detecting 
-	## and logging the service for this connection is complete.
+	# and logging the service for this connection is complete.
 	known_services_done: bool &default=F;
 };