Merge branch 'topic/jgras/intel-filter' of https://github.com/J-Gras/zeek

* 'topic/jgras/intel-filter' of https://github.com/J-Gras/zeek:
  Added new intel policy script to policy test.
  Added test for intel removal policy script.
  Added policy script for intel removal.
  Added test for intel item filtering.
  Added hook to filter intelligence items.

CHANGES

@@ -1,4 +1,10 @@
 
+2.6-186 | 2019-03-25 09:41:57 -0700
+
+  * Added policy script for intel removal. (Jan Grashoefer)
+
+  * Added Intel::filter_item hook to filter intelligence items. (Jan Grashoefer)
+
 2.6-178 | 2019-03-21 14:10:44 -0700
 
   * Add support for parsing SMB 3.1.1 NegotiateContextList response values (Mauro Palumbo)

NEWS

@@ -61,6 +61,9 @@ New Functionality
     containing capability information found in an SMB 3.1.1 dialect's
     negotiation message.
 
+- Added a new hook, ``Intel::filter_item``, to assist in filtering and
+  removal of intelligence items that are about to be inserted.
+
 Changed Functionality
 ---------------------
 

VERSION

@@ -1 +1 @@
-2.6-178
+2.6-186

doc

@@ -1 +1 @@
-Subproject commit ee040c581f8a4e1ea82a7ec5f17d5f36f1184324
+Subproject commit ae04aa61a80ff4f436a9c248f9a934a694bb500a

scripts/base/frameworks/intel/main.bro

@@ -173,6 +173,14 @@ export {
 	## be removed.
 	global item_expired: hook(indicator: string, indicator_type: Type, metas: set[MetaData]);
 
+	## This hook can be used to filter intelligence items that are about to be
+	## inserted into the internal data store. In case the hook execution is
+	## terminated using break, the item will not be (re)added to the internal
+	## data store.
+	##
+	## item: The intel item that should be inserted.
+	global filter_item: hook(item: Intel::Item);
+
 	global log_intel: event(rec: Info);
 }
 
@@ -495,10 +503,13 @@ function _insert(item: Item, first_dispatch: bool &default = T)
 	}
 
 function insert(item: Item)
+	{
+	if ( hook filter_item(item) )
 		{
 		# Insert possibly new item.
 		_insert(item, T);
 		}
+	}
 
 # Function to check whether an item is present.
 function item_exists(item: Item): bool

scripts/policy/frameworks/intel/removal.bro

@@ -0,0 +1,23 @@
+##! This script enables removal of intelligence items.
+
+@load base/frameworks/intel
+
+module Intel;
+
+export {
+	redef record Intel::MetaData += {
+		## A boolean value to indicate whether the item should be removed.
+		remove: bool &default=F;
+	};
+}
+
+hook Intel::filter_item(item: Item)
+	{
+	if ( item$meta$remove )
+		{
+		Intel::remove(item);
+		# Prevent readding
+		break;
+		}
+	}
+

scripts/test-all-policy.bro

@@ -16,6 +16,7 @@
 @load frameworks/intel/do_notice.bro
 @load frameworks/intel/do_expire.bro
 @load frameworks/intel/whitelist.bro
+@load frameworks/intel/removal.bro
 @load frameworks/intel/seen/__load__.bro
 @load frameworks/intel/seen/conn-established.bro
 @load frameworks/intel/seen/dns.bro

testing/btest/Baseline/scripts.base.frameworks.intel.filter-item/broproc.intel.log

@@ -0,0 +1,10 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	intel
+#open	2019-03-24-20-29-18
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources	fuid	file_mime_type	file_desc
+#types	time	string	addr	port	addr	port	string	enum	enum	string	set[enum]	set[string]	string	string	string
+1553459358.205227	-	-	-	-	-	1.2.3.42	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1	-	-	-
+#close	2019-03-24-20-29-18

testing/btest/Baseline/scripts.base.frameworks.intel.remove-non-existing/output

@@ -3,9 +3,9 @@
 #empty_field	(empty)
 #unset_field	-
 #path	reporter
-#open	2018-02-27-17-25-30
+#open	2019-03-24-20-20-10
 #fields	ts	level	message	location
 #types	time	enum	string	string
-0.000000	Reporter::INFO	Tried to remove non-existing item '192.168.1.1' (Intel::ADDR).	/home/jgras/devel/bro/scripts/base/frameworks/intel/./main.bro, lines 552-553
+0.000000	Reporter::INFO	Tried to remove non-existing item '192.168.1.1' (Intel::ADDR).	/home/jgras/devel/zeek/scripts/base/frameworks/intel/./main.bro, lines 563-564
 0.000000	Reporter::INFO	received termination signal	(empty)
-#close	2018-02-27-17-25-30
+#close	2019-03-24-20-20-10

testing/btest/Baseline/scripts.policy.frameworks.intel.removal/broproc.intel.log

@@ -0,0 +1,10 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	intel
+#open	2019-03-24-21-15-06
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources	fuid	file_mime_type	file_desc
+#types	time	string	addr	port	addr	port	string	enum	enum	string	set[enum]	set[string]	string	string	string
+1553462106.131323	-	-	-	-	-	10.0.0.2	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1	-	-	-
+#close	2019-03-24-21-15-06

testing/btest/scripts/base/frameworks/intel/filter-item.bro

@@ -0,0 +1,43 @@
+
+# @TEST-EXEC: btest-bg-run broproc bro %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff broproc/intel.log
+
+@TEST-START-FILE intel.dat
+#fields	indicator	indicator_type	meta.source	meta.desc	meta.url
+1.2.3.42	Intel::ADDR	source1	this host is just plain baaad	http://some-data-distributor.com/1234
+10.0.0.1	Intel::ADDR	source1	this host is just plain baaad	http://some-data-distributor.com/1234
+@TEST-END-FILE
+
+redef exit_only_after_terminate = T;
+redef Site::local_nets += { 10.0.0.0/8 };
+redef Intel::read_files += { "../intel.dat" };
+redef enum Intel::Where += { SOMEWHERE };
+
+hook Intel::filter_item(item: Intel::Item)
+	{
+	if ( item$indicator_type == Intel::ADDR &&
+		 Site::is_local_addr(to_addr(item$indicator)) )
+		break;
+	}
+
+event do_it()
+	{
+	Intel::seen([$host=10.0.0.1,
+	             $where=SOMEWHERE]);
+	Intel::seen([$host=1.2.3.42,
+	             $where=SOMEWHERE]);
+	}
+
+global log_lines = 0;
+event Intel::log_intel(rec: Intel::Info)
+	{
+	++log_lines;
+	if ( log_lines == 1 )
+		terminate();
+	}
+
+event bro_init() &priority=-10
+	{
+	schedule 1sec { do_it() };
+	}

testing/btest/scripts/policy/frameworks/intel/removal.bro

@@ -0,0 +1,46 @@
+
+# @TEST-EXEC: btest-bg-run broproc bro %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff broproc/intel.log
+
+@TEST-START-FILE intel.dat
+#fields	indicator	indicator_type	meta.source	meta.remove
+10.0.0.1	Intel::ADDR	source1	T
+10.0.0.2	Intel::ADDR	source1	F
+@TEST-END-FILE
+
+@load frameworks/intel/removal
+
+redef exit_only_after_terminate = T;
+redef Intel::read_files += { "../intel.dat" };
+redef enum Intel::Where += { SOMEWHERE };
+
+hook Intel::filter_item(item: Intel::Item)
+	{
+	if ( item$indicator_type == Intel::ADDR &&
+		 Site::is_local_addr(to_addr(item$indicator)) )
+		break;
+	}
+
+event do_it()
+	{
+	Intel::seen([$host=10.0.0.1,
+	             $where=SOMEWHERE]);
+	Intel::seen([$host=10.0.0.2,
+	             $where=SOMEWHERE]);
+	}
+
+global log_lines = 0;
+event Intel::log_intel(rec: Intel::Info)
+	{
+	++log_lines;
+	if ( log_lines == 1 )
+		terminate();
+	}
+
+event bro_init() &priority=-10
+	{
+	Intel::insert([$indicator="10.0.0.1", $indicator_type=Intel::ADDR, $meta=[$source="source1"]]);
+	Intel::insert([$indicator="10.0.0.2", $indicator_type=Intel::ADDR, $meta=[$source="source1"]]);
+	schedule 1sec { do_it() };
+	}