Merge branch 'topic/johanna/GH-169'

* topic/johanna/GH-169:
  Make event ordering deterministic
  dump-events: try to make baseline work on all systems
  Introduce generate_all_events bif and add option to misc/dump-events

Fixes GH-169

CHANGES

@@ -1,3 +1,19 @@
+
+3.3.0-dev.451 | 2020-10-16 07:09:43 +0000
+
+  * Make event ordering deterministic
+
+    NetControl::init and filter_change_tracking could basically be raised in
+    random order. (Johanna Amann, Corelight)
+
+  * Introduce generate_all_events bif and add option to misc/dump-events
+
+    generate_all_events causes all events to be raised internally; this
+    makes it possible for dump_events to really capture all events (and not
+    just those that were handled).
+
+    Addresses GH-169 (Johanna Amann, Corelight)
+
 3.3.0-dev.444 | 2020-10-15 13:25:12 -0700
 
   * Rework Sessions::Weird (Tim Wojtulewicz, Corelight)

NEWS

@@ -61,6 +61,11 @@ New Functionality
 
     #!/usr/local/zeek/bin/zeek --
 
+- Added a new ``generate_all_events`` bif, which can be used to always raise
+  events, even when they are not used by scripts. This can be used by the
+  ``dump-events.zeek`` script to log all events that happen; the script
+  got a new option to enable this behavior.
+
 Changed Functionality
 ---------------------
 

VERSION

@@ -1 +1 @@
-3.3.0-dev.444
+3.3.0-dev.451

scripts/base/frameworks/packet-filter/main.zeek

@@ -178,7 +178,7 @@ event zeek_init() &priority=5
 		}
 	}
 
-event zeek_init() &priority=-5
+event zeek_init() &priority=-6
 	{
 	install();
 

scripts/policy/misc/dump-events.zeek

@@ -9,11 +9,21 @@ export {
 	## If true, include event arguments in output.
 	option include_args = T;
 
+	## By default, only events that are handled in a script are dumped. Setting this option to true
+	## will cause unhandled events to be dumped too.
+	const dump_all_events = F &redef;
+
 	## Only include events matching the given pattern into output. By default, the
 	## pattern matches all events.
 	option include = /.*/;
 }
 
+event zeek_init() &priority=999
+	{
+	if ( dump_all_events )
+		generate_all_events();
+	}
+
 event new_event(name: string, args: call_argument_vector)
 	{
 	if ( include !in name )

src/zeek.bif

@@ -5046,6 +5046,27 @@ function match_signatures%(c: connection, pattern_type: int, s: string,
 	return zeek::val_mgr->True();
 	%}
 
+## By default, zeek does not generate (raise) events that have not handled by
+## any scripts. This means that these events will be invisible to a lot of other
+## event handlers - and will not raise :zeek:id:`new_event`.
+##
+## Calling this function will cause all event handlers to be raised. This is, likely,
+## only useful for debugging and causes reduced performance.
+function generate_all_events%(%) : bool
+	%{
+	auto event_names = event_registry->AllHandlers();
+	for ( const auto& name: event_names )
+		{
+		auto event = event_registry->Lookup(name);
+		if ( event == nullptr )
+			continue;
+
+		event->SetGenerateAlways();
+		}
+
+	return zeek::val_mgr->True();
+	%}
+
 %%{
 // Autogenerated from CMake bif_target()
 #include "__all__.bif.cc"

testing/btest/Baseline/scripts.policy.misc.dump-events/all-events-no-args.log

@@ -1,6 +1,6 @@
          0.000000 zeek_init
-         0.000000 filter_change_tracking
          0.000000 NetControl::init
+         0.000000 filter_change_tracking
 1254722767.492060 Broker::log_flush
 1254722767.492060 ChecksumOffloading::check
 1254722767.492060 filter_change_tracking

testing/btest/Baseline/scripts.policy.misc.dump-events/all-events.log

@@ -1,6 +1,6 @@
          0.000000 zeek_init
-         0.000000 filter_change_tracking
          0.000000 NetControl::init
+         0.000000 filter_change_tracking
 1254722767.492060 Broker::log_flush
 1254722767.492060 ChecksumOffloading::check
 1254722767.492060 filter_change_tracking

testing/btest/scripts/policy/misc/dump-events.zeek

@@ -1,10 +1,12 @@
 # @TEST-EXEC: zeek -r $TRACES/smtp.trace policy/misc/dump-events %INPUT >all-events.log
 # @TEST-EXEC: zeek -r $TRACES/smtp.trace policy/misc/dump-events %INPUT DumpEvents::include_args=F >all-events-no-args.log
 # @TEST-EXEC: zeek -r $TRACES/smtp.trace policy/misc/dump-events %INPUT DumpEvents::include=/smtp_/ >smtp-events.log
+# @TEST-EXEC: zeek -r $TRACES/smtp.trace policy/misc/dump-events %INPUT DumpEvents::dump_all_events=T | grep -v "CPU: interval\|samples: set\|path: string" > really-all-events.log
 #
 # @TEST-EXEC: btest-diff all-events.log
 # @TEST-EXEC: btest-diff all-events-no-args.log
 # @TEST-EXEC: btest-diff smtp-events.log
+# @TEST-EXEC: btest-diff really-all-events.log
 
 # There is some kind of race condition between the MD5 and SHA1 events, which are added
 # by the SSL parser. Just remove MD5, this is not important for this test.