Merge branch 'master' of ssh://git.bro.org/bro into topic/vladg/radius

scripts/base/frameworks/reporter/main.bro

@@ -9,7 +9,7 @@
 ##! Note that this framework deals with the handling of internally generated
 ##! reporter messages, for the interface
 ##! into actually creating reporter messages from the scripting layer, use
-##! the built-in functions in :doc:`/scripts/base/bif/reporter.bif`.
+##! the built-in functions in :doc:`/scripts/base/bif/reporter.bif.bro`.
 
 module Reporter;
 

scripts/base/frameworks/software/main.bro

@@ -209,7 +209,7 @@ function parse_mozilla(unparsed_version: string): Description
 		if ( 2 in parts )
 			v = parse(parts[2])$version;
 		}
-	else if ( / MSIE / in unparsed_version )
+	else if ( / MSIE |Trident\// in unparsed_version )
 		{
 		software_name = "MSIE";
 		if ( /Trident\/4\.0/ in unparsed_version )
@@ -218,6 +218,8 @@ function parse_mozilla(unparsed_version: string): Description
 			v = [$major=9,$minor=0];
 		else if ( /Trident\/6\.0/ in unparsed_version )
 			v = [$major=10,$minor=0];
+		else if ( /Trident\/7\.0/ in unparsed_version )
+			v = [$major=11,$minor=0];
 		else
 			{
 			parts = split_all(unparsed_version, /MSIE [0-9]{1,2}\.*[0-9]*b?[0-9]*/);

scripts/base/init-bare.bro

@@ -399,7 +399,7 @@ type NetStats: record {
 	pkts_dropped: count &default=0;	##< Packets reported dropped by the system.
 	## Packets seen on the link. Note that this may differ
 	## from *pkts_recvd* because of a potential capture_filter. See
-	## :doc:`/scripts/base/frameworks/packet-filter/main`. Depending on the
+	## :doc:`/scripts/base/frameworks/packet-filter/main.bro`. Depending on the
 	## packet capture system, this value may not be available and will then
 	## be always set to zero.
 	pkts_link:    count &default=0;
@@ -507,7 +507,7 @@ type script_id: record {
 ##    directly and then remove this alias.
 type id_table: table[string] of script_id;
 
-## Meta-information about a record-field.
+## Meta-information about a record field.
 ##
 ## .. bro:see:: record_fields record_field_table
 type record_field: record {
@@ -529,6 +529,25 @@ type record_field: record {
 ##    directly and then remove this alias.
 type record_field_table: table[string] of record_field;
 
+## Meta-information about a parameter to a function/event.
+##
+## .. bro:see:: call_argument_vector new_event
+type call_argument: record {
+	name: string;	##< The name of the parameter.
+	type_name: string;	##< The name of the parameters's type.
+	default_val: any &optional;	##< The value of the :bro:attr:`&default` attribute if defined.
+
+	## The value of the parameter as passed into a given call instance.
+	## Might be unset in the case a :bro:attr:`&default` attribute is
+	## defined.
+	value: any &optional;
+};
+
+## Vector type used to capture parameters of a function/event call.
+##
+## .. bro:see:: call_argument new_event
+type call_argument_vector: vector of call_argument;
+
 # todo:: Do we still need these here? Can they move into the packet filter
 # framework?
 #
@@ -2768,13 +2787,13 @@ const log_max_size = 0.0 &redef;
 const log_encryption_key = "<undefined>" &redef;
 
 ## Write profiling info into this file in regular intervals. The easiest way to
-## activate profiling is loading :doc:`/scripts/policy/misc/profiling`.
+## activate profiling is loading :doc:`/scripts/policy/misc/profiling.bro`.
 ##
 ## .. bro:see:: profiling_interval expensive_profiling_multiple segment_profiling
 global profiling_file: file &redef;
 
 ## Update interval for profiling (0 disables).  The easiest way to activate
-## profiling is loading  :doc:`/scripts/policy/misc/profiling`.
+## profiling is loading  :doc:`/scripts/policy/misc/profiling.bro`.
 ##
 ## .. bro:see:: profiling_file expensive_profiling_multiple segment_profiling
 const profiling_interval = 0 secs &redef;

scripts/base/protocols/dhcp/main.bro

@@ -4,7 +4,7 @@
 ##!
 ##! If you'd like to track known DHCP devices and to log the hostname
 ##! supplied by the client, see
-##! :doc:`/scripts/policy/protocols/dhcp/known-devices-and-hostnames`.
+##! :doc:`/scripts/policy/protocols/dhcp/known-devices-and-hostnames.bro`.
 
 @load ./utils.bro
 

scripts/base/protocols/http/__load__.bro

@@ -3,4 +3,4 @@
 @load ./utils
 @load ./files
 
-@load-sigs ./dpd.sig
+@load-sigs ./dpd.sig

scripts/base/protocols/smtp/main.bro

@@ -291,7 +291,7 @@ function describe(rec: Info): string
 			{
 			if ( |rec$subject| > 20 )
 				{
-				abbrev_subject = rec$subject[0:20] + "...";
+				abbrev_subject = rec$subject[0:21] + "...";
 				}
 			}
 

scripts/base/protocols/ssl/main.bro

@@ -153,7 +153,7 @@ function finish(c: connection)
 		disable_analyzer(c$id, c$ssl$analyzer_id);
 	}
 
-event ssl_client_hello(c: connection, version: count, possible_ts: time, client_random: string, session_id: string, ciphers: count_set) &priority=5
+event ssl_client_hello(c: connection, version: count, possible_ts: time, client_random: string, session_id: string, ciphers: index_vec) &priority=5
 	{
 	set_session(c);
 

scripts/broxygen/README

@@ -0,0 +1,4 @@
+This package is loaded during the process which automatically generates
+reference documentation for all Bro scripts (i.e. "Broxygen").  Its only
+purpose is to provide an easy way to load all known Bro scripts plus any
+extra scripts needed or used by the documentation process.

scripts/broxygen/__load__.bro

@@ -0,0 +1,15 @@
+@load test-all-policy.bro
+
+# Scripts which are commented out in test-all-policy.bro.
+@load protocols/ssl/notary.bro
+@load frameworks/communication/listen.bro
+@load frameworks/control/controllee.bro
+@load frameworks/control/controller.bro
+@load policy/misc/dump-events.bro
+
+@load ./example.bro
+
+event bro_init()
+	{
+	terminate();
+	}

scripts/broxygen/example.bro

@@ -0,0 +1,194 @@
+##! This is an example script that demonstrates Broxygen-style
+##! documentation.  It generally will make most sense when viewing
+##! the script's raw source code and comparing to the HTML-rendered
+##! version.
+##!
+##! Comments in the from ``##!`` are meant to summarize the script's
+##! purpose.  They are transferred directly in to the generated
+##! `reStructuredText <http://docutils.sourceforge.net/rst.html>`_
+##! (reST) document associated with the script.
+##!
+##! .. tip:: You can embed directives and roles within ``##``-stylized comments.
+##!
+##! There's also a custom role to reference any identifier node in
+##! the Bro Sphinx domain that's good for "see alsos", e.g.
+##!
+##! See also: :bro:see:`BroxygenExample::a_var`,
+##! :bro:see:`BroxygenExample::ONE`, :bro:see:`SSH::Info`
+##!
+##! And a custom directive does the equivalent references:
+##!
+##! .. bro:see:: BroxygenExample::a_var BroxygenExample::ONE SSH::Info
+
+# Comments that use a single pound sign (#) are not significant to
+# a script's auto-generated documentation, but ones that use a
+# double pound sign (##) do matter.  In some cases, like record
+# field comments, it's necessary to disambiguate the field with
+# which a comment associates: e.g. "##<" can be used on the same line
+# as a field to signify the comment relates to it and not the
+# following field. "##<" can also be used more generally in any
+# variable declarations to associate with the last-declared identifier.
+#
+# Generally, the auto-doc comments (##) are associated with the
+# next declaration/identifier found in the script, but Broxygen
+# will track/render identifiers regardless of whether they have any
+# of these special comments associated with them.
+#
+# The first sentence contained within the "##"-stylized comments for
+# a given identifier is special in that it will be used as summary
+# text in a table containing all such identifiers and short summaries.
+# If there are no sentences (text terminated with '.'), then everything
+# in the "##"-stylized comments up until the first empty comment
+# is taken as the summary text for a given identifier.
+
+# @load directives are self-documenting, don't use any ``##`` style
+# comments with them.
+@load base/frameworks/notice
+@load base/protocols/http
+@load frameworks/software/vulnerable
+
+# "module" statements are self-documenting, don't use any ``##`` style
+# comments with them.
+module BroxygenExample;
+
+# Redefinitions of "Notice::Type" are self-documenting, but
+# more information can be supplied in two different ways.
+redef enum Notice::Type += {
+	## Any number of this type of comment
+	## will document "Broxygen_One".
+	Broxygen_One,
+	Broxygen_Two,  ##< Any number of this type of comment
+	               ##< will document "BROXYGEN_TWO".
+	Broxygen_Three,
+	## Omitting comments is fine, and so is mixing ``##`` and ``##<``, but
+	Broxygen_Four,  ##< it's probably best to use only one style consistently.
+};
+
+# All redefs are automatically tracked.  Comments of the "##" form can be use
+# to further document it, but in some cases, like here, they wouldn't be
+# ading any interesting information that's not implicit.
+redef enum Log::ID += { LOG };
+
+# Only identifiers declared in an export section will show up in generated docs.
+
+export {
+
+	## Documentation for the "SimpleEnum" type goes here.
+	## It can span multiple lines.
+	type SimpleEnum: enum {
+		## Documentation for particular enum values is added like this.
+		## And can also span multiple lines.
+		ONE,
+		TWO,  ##< Or this style is valid to document the preceding enum value.
+		THREE,
+	};
+
+	## Document the "SimpleEnum" redef here with any special info regarding
+	## the *redef* itself.
+	redef enum SimpleEnum  += {
+		FOUR, ##< And some documentation for "FOUR".
+		## Also "FIVE".
+		FIVE
+	};
+
+	## General documentation for a type "SimpleRecord" goes here.
+	## The way fields can be documented is similar to what's already seen
+	## for enums.
+	type SimpleRecord: record {
+		## Counts something.
+		field1: count;
+		field2: bool; ##< Toggles something.
+	};
+
+	## Document the record extension *redef* itself here.
+	redef record SimpleRecord += {
+		## Document the extending field like this.
+		field_ext: string &optional; ##< Or here, like this.
+	};
+
+	## General documentation for a type "ComplexRecord" goes here.
+	type ComplexRecord: record {
+		field1: count;               ##< Counts something.
+		field2: bool;                ##< Toggles something.
+		field3: SimpleRecord;        ##< Broxygen automatically tracks types
+		                             ##< and cross-references are automatically
+		                             ##< inserted in to generated docs.
+		msg: string &default="blah"; ##< Attributes are self-documenting.
+	} &redef;
+
+	## An example record to be used with a logging stream.
+	## Nothing special about it.  If another script redefs this type
+	## to add fields, the generated documentation will show all original
+	## fields plus the extensions and the scripts which contributed to it
+	## (provided they are also @load'ed).
+	type Info: record {
+		ts:       time       &log;
+		uid:      string     &log;
+		status:   count      &log &optional;
+	};
+
+	## Add documentation for "an_option" here.
+	## The type/attribute information is all generated automatically.
+	const an_option: set[addr, addr, string] &redef;
+
+	## Default initialization will be generated automatically.
+	const option_with_init = 0.01 secs &redef; ##< More docs can be added here.
+
+	## Put some documentation for "a_var" here.  Any global/non-const that
+	## isn't a function/event/hook is classified as a "state variable"
+	## in the generated docs.
+	global a_var: bool;
+
+	## Types are inferred, that information is self-documenting.
+	global var_without_explicit_type = "this works";
+
+	## The first sentence for a particular identifier's summary text ends here.
+	## And this second sentence doesn't show in the short description provided
+	## by the table of all identifiers declared by this script.
+	global summary_test: string;
+
+    ## Summarize purpose of "a_function" here.
+    ## Give more details about "a_function" here.
+    ## Separating the documentation of the params/return values with
+    ## empty comments is optional, but improves readability of script.
+    ##
+    ## tag: Function arguments can be described
+    ##      like this.
+	##
+    ## msg: Another param.
+    ##
+    ## Returns: Describe the return type here.
+    global a_function: function(tag: string, msg: string): string;
+
+    ## Summarize "an_event" here.
+    ## Give more details about "an_event" here.
+	##
+	## BroxygenExample::a_function should not be confused as a parameter
+	## in the generated docs, but it also doesn't generate a cross-reference
+	## link.  Use the see role instead: :bro:see:`BroxygenExample::a_function`.
+	##
+    ## name: Describe the argument here.
+    global an_event: event(name: string);
+}
+
+# This function isn't exported, so it won't appear anywhere in the generated
+# documentation.  So using ``##``-style comments is pointless here.
+function function_without_proto(tag: string): string
+    {
+    return "blah";
+    }
+
+# Same thing goes for types -- it's not exported, so it's considered
+# private to this script and comments are only interesting to a person
+# who is already reading the raw source for the script (so don't use
+# ``##`` comments here.
+type PrivateRecord: record {
+    field1: bool;
+    field2: count;
+};
+
+# Event handlers are also an implementation detail of a script, so they
+# don't show up anywhere in the generated documentation.
+event bro_init()
+    {
+    }

scripts/policy/misc/dump-events.bro

@@ -0,0 +1,40 @@
+##! This script dumps the events that Bro raises out to standard output in a
+##! readable form. This is for debugging only and allows to understand events and
+##! their parameters as Bro processes input. Note that it will show only events
+##! for which a handler is defined.
+
+module DumpEvents;
+
+export {
+	## If true, include event arguments in output.
+	const include_args = T &redef;
+
+	## Only include events matching the given pattern into output. By default, the
+	## pattern matches all events.
+	const include = /.*/ &redef;
+}
+
+event new_event(name: string, args: call_argument_vector)
+	{
+	if ( include !in name )
+		return;
+
+	print fmt("%17.6f %s", network_time(), name);
+
+	if ( ! include_args || |args| == 0 )
+		return;
+
+	for ( i in args )
+		{
+		local a = args[i];
+
+		local proto = fmt("%s: %s", a$name, a$type_name);
+
+		if ( a?$value )
+			print fmt("                  [%d] %-18s = %s", i, proto, a$value);
+		else
+			print fmt("                  | %-18s = %s [default]", proto, a$value);
+		}
+
+	print "";
+	}

scripts/policy/misc/known-devices.bro

@@ -7,7 +7,7 @@
 ##!
 ##!     This script will not generate any logs on its own, it needs to be
 ##!     supplied with information from elsewhere, such as
-##!     :doc:`/scripts/policy/protocols/dhcp/known-devices-and-hostnames`.
+##!     :doc:`/scripts/policy/protocols/dhcp/known-devices-and-hostnames.bro`.
 
 module Known;
 

scripts/policy/misc/scan.bro

@@ -147,11 +147,6 @@ function is_reverse_failed_conn(c: connection): bool
 	return F;
 	}
 
-## Generated for an unsuccessful connection attempt. This
-## event is raised when an originator unsuccessfully attempted
-## to establish a connection. "Unsuccessful" is defined as at least
-## tcp_attempt_delay seconds having elapsed since the originator first sent a
-## connection establishment packet to the destination without seeing a reply.
 event connection_attempt(c: connection)
 	{
 	local is_reverse_scan = F;
@@ -161,9 +156,6 @@ event connection_attempt(c: connection)
 	add_sumstats(c$id, is_reverse_scan);
 	}
 
-## Generated for a rejected TCP connection. This event is raised when an
-## originator attempted to setup a TCP connection but the responder replied with
-## a RST packet denying it.
 event connection_rejected(c: connection)
 	{
 	local is_reverse_scan = F;
@@ -173,9 +165,6 @@ event connection_rejected(c: connection)
 	add_sumstats(c$id, is_reverse_scan);
 	}
 
-## Generated when an endpoint aborted a TCP connection. The event is raised when
-## one endpoint of an *established* TCP connection aborted by sending a RST
-## packet.
 event connection_reset(c: connection)
 	{
 	if ( is_failed_conn(c) )
@@ -184,7 +173,6 @@ event connection_reset(c: connection)
 		add_sumstats(c$id, T);
 	}
 
-## Generated for each still-open connection when Bro terminates.
 event connection_pending(c: connection)
 	{
 	if ( is_failed_conn(c) )

scripts/policy/misc/stats.bro

@@ -1,5 +1,5 @@
 ##! Log memory/packet/lag statistics.  Differs from
-##! :doc:`/scripts/policy/misc/profiling` in that this
+##! :doc:`/scripts/policy/misc/profiling.bro` in that this
 ##! is lighter-weight (much less info, and less load to generate).
 
 @load base/frameworks/notice

scripts/test-all-policy.bro

@@ -48,6 +48,7 @@
 @load misc/capture-loss.bro
 @load misc/detect-traceroute/__load__.bro
 @load misc/detect-traceroute/main.bro
+# @load misc/dump-events.bro
 @load misc/known-devices.bro
 @load misc/load-balancing.bro
 @load misc/loaded-scripts.bro