Merge branch 'J-Gras-topic/jgras/bit-1679'

* J-Gras-topic/jgras/bit-1679:
  Handle removing non-existent intel items.
  Separated file and default info added to matches.

CHANGES

@@ -1,4 +1,8 @@
 
+2.5-beta-33 | 2016-10-02 14:42:22 -0400
+
+  * Handle removing non-existent intel items. (Jan Grashoefer)
+
 2.5-beta-29 | 2016-09-28 18:18:35 -0700
 
   * Prettifying reporter output in case no epxression is associated

VERSION

@@ -1 +1 @@
-2.5-beta-29
+2.5-beta-33

scripts/base/frameworks/intel/files.bro

@@ -45,7 +45,7 @@ export {
 }
 
 # Add file information to matches if available.
-hook extend_match(info: Info, s: Seen, items: set[Item]) &priority=5
+hook extend_match(info: Info, s: Seen, items: set[Item]) &priority=6
 	{
 	if ( s?$f )
 		{
@@ -66,19 +66,4 @@ hook extend_match(info: Info, s: Seen, items: set[Item]) &priority=5
 
 	if ( s?$fuid )
 		info$fuid = s$fuid;
-
-	if ( s?$conn )
-		{
-		s$uid = s$conn$uid;
-		info$id  = s$conn$id;
-		}
-
-	if ( s?$uid )
-		info$uid = s$uid;
-
-	for ( item in items )
-		{
-		add info$sources[item$meta$source];
-		add info$matched[item$indicator_type];
-		}
 	}

scripts/base/frameworks/intel/main.bro

@@ -369,6 +369,25 @@ event Intel::match(s: Seen, items: set[Item]) &priority=5
 		Log::write(Intel::LOG, info);
 	}
 
+hook extend_match(info: Info, s: Seen, items: set[Item]) &priority=5
+	{
+	# Add default information to matches.
+	if ( s?$conn )
+		{
+		s$uid = s$conn$uid;
+		info$id  = s$conn$id;
+		}
+
+	if ( s?$uid )
+		info$uid = s$uid;
+
+	for ( item in items )
+		{
+		add info$sources[item$meta$source];
+		add info$matched[item$indicator_type];
+		}
+	}
+
 function insert(item: Item)
 	{
 	# Create and fill out the metadata item.
@@ -436,6 +455,22 @@ function insert(item: Item)
 		event Intel::new_item(item);
 	}
 
+# Function to check whether an item is present.
+function item_exists(item: Item): bool
+	{
+	local ds = have_full_data ? data_store : min_data_store;
+
+	switch ( item$indicator_type )
+		{
+		case ADDR:
+			return to_addr(item$indicator) in ds$host_data;
+		case SUBNET:
+			return to_subnet(item$indicator) in ds$subnet_data;
+		default:
+			return [item$indicator, item$indicator_type] in ds$string_data;
+		}
+	}
+
 # Function to remove metadata of an item. The function returns T
 # if there is no metadata left for the given indicator.
 function remove_meta_data(item: Item): bool
@@ -465,6 +500,14 @@ function remove_meta_data(item: Item): bool
 
 function remove(item: Item, purge_indicator: bool)
 	{
+	# Check whether the indicator is present
+	if ( ! item_exists(item) )
+		{
+		Reporter::info(fmt("Tried to remove non-existing item '%s' (%s).",
+			item$indicator, item$indicator_type));
+		return;
+		}
+
 	# Delegate removal if we are on a worker
 	if ( !have_full_data )
 		{

testing/btest/Baseline/scripts.base.frameworks.intel.remove-non-existing/output

@@ -0,0 +1,11 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	reporter
+#open	2016-09-20-22-35-58
+#fields	ts	level	message	location
+#types	time	enum	string	string
+0.000000	Reporter::INFO	Tried to remove non-existing item '192.168.1.1' (Intel::ADDR).	/home/jgras/devel/bro/scripts/base/frameworks/intel/./main.bro, lines 506-507
+0.000000	Reporter::INFO	received termination signal	(empty)
+#close	2016-09-20-22-35-59

testing/btest/scripts/base/frameworks/intel/remove-non-existing.bro

@@ -0,0 +1,31 @@
+# @TEST-EXEC: btest-bg-run broproc bro %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: cat broproc/reporter.log > output
+# @TEST-EXEC: cat broproc/.stdout >> output
+# @TEST-EXEC: TEST_DIFF_CANONIFIER="$SCRIPTS/diff-remove-abspath | $SCRIPTS/diff-remove-timestamps" btest-diff output
+
+# @TEST-START-FILE intel.dat
+#fields	indicator	indicator_type	meta.source	meta.desc	meta.url
+192.168.1.1	Intel::ADDR	source1	this host is just plain baaad	http://some-data-distributor.com/1
+# @TEST-END-FILE
+
+@load frameworks/communication/listen
+
+redef Intel::read_files += { "../intel.dat" };
+redef enum Intel::Where += { SOMEWHERE };
+
+event do_it()
+	{
+	# not existing meta data:
+	Intel::remove([$indicator="192.168.1.1", $indicator_type=Intel::ADDR, $meta=[$source="source23"]]);
+	# existing:
+	Intel::remove([$indicator="192.168.1.1", $indicator_type=Intel::ADDR, $meta=[$source="source1"]]);
+	# not existing item:
+	Intel::remove([$indicator="192.168.1.1", $indicator_type=Intel::ADDR, $meta=[$source="source1"]]);
+	terminate();
+	}
+
+event bro_init() &priority=-10
+	{
+	schedule 1sec { do_it() };
+	}