Finished work on unified2 analyzer.

scripts/base/files/unified2/main.bro

@@ -5,31 +5,148 @@
 module Unified2;
 
 export {
+	redef enum Log::ID += { LOG };
+
 	## Directory to watch for Unified2 files.
 	const watch_file = "" &redef;
 
 	## File to watch for Unified2 records.
 	const watch_dir = "" &redef;
 
+	## The sid-msg.map file you would like to use for your alerts.
+	const sid_msg = "" &redef;
+
+	## The gen-msg.map file you would like to use for your alerts.
+	const gen_msg = "" &redef;
+
+	## The classification.config file you would like to use for your alerts.
+	const classification_config = "" &redef;
+
 	## Reconstructed "alert" which combines related events
 	## and packets.
 	global alert: event(f: fa_file, ev: Unified2::IDSEvent, pkt: Unified2::Packet);
 
+	type PacketID: record {
+		src_ip: addr;
+		src_p: port;
+		dst_ip: addr;
+		dst_p: port;
+	} &log;
+
 	type Info: record {
-		## The last received IDS event.  This is primarily used 
+		## Timestamp attached to the alert.
-		## for tying together Unified2 events and packets.
+		ts:                 time     &log; 
-		current_event: Unified2::IDSEvent &optional;
+		## Addresses and ports for the connection.
-	};
+		id:                 PacketID &log; 
+		## Sensor that originated this event.
+		sensor_id:          count    &log; 
+		## Sig id for this generator.
+		signature_id:       count    &log; 
+		## A string representation of the "signature_id" field if a sid_msg.map file was loaded.
+		signature:          string   &log &optional;
+		## Which generator generated the alert?
+		generator_id:       count    &log; 
+		## A string representation of the "generator_id" field if a gen_msg.map file was loaded.
+		generator:          string   &log &optional; 
+		## Sig revision for this id.
+		signature_revision: count    &log; 
+		## Event classification.
+		classification_id:  count    &log;
+		## Descriptive classification string,
+		classification:     string   &log &optional;
+		## Event priority.
+		priority_id:        count    &log; 
+		## Event ID.
+		event_id:           count    &log; 
+		## Some of the packet data.
+		packet:             string   &log; 
+	} &log;
 
 	redef record fa_file += {
-		## Add a field to store per-file state about Unified2
+		## Add a field to store per-file state about Unified2 files.
-		## files.
 		unified2: Info &optional;
+
+		## Recently received IDS events.  This is primarily used 
+		## for tying together Unified2 events and packets.
+		u2_events: table[count] of Unified2::IDSEvent &optional &create_expire=5sec;
 	};
+
+	## The event for accessing logged records.
+	global log_unified2: event(rec: Info);
 }
 
+# Mappings for extended information from alerts.
+global classification_map: table[count] of string;
+global sid_map: table[count] of string;
+global gen_map: table[count] of string;
+
+# For reading in config files.
+type OneLine: record {
+	line: string;
+};
+
+event Unified2::read_sid_msg_line(desc: Input::EventDescription, tpe: Input::Event, line: string)
+	{
+	local parts = split_n(line, / \|\| /, F, 100);
+	if ( |parts| >= 2 && /^[0-9]+$/ in parts[1] )
+		sid_map[to_count(parts[1])] = parts[2];
+	}
+
+event Unified2::read_gen_msg_line(desc: Input::EventDescription, tpe: Input::Event, line: string)
+	{
+	local parts = split_n(line, / \|\| /, F, 3);
+	if ( |parts| >= 2 && /^[0-9]+$/ in parts[1] )
+		gen_map[to_count(parts[1])] = parts[3];
+	}
+
+event Unified2::read_classification_line(desc: Input::EventDescription, tpe: Input::Event, line: string)
+	{
+	local parts = split_n(line, /: /, F, 2);
+	if ( |parts| == 2 )
+		{
+		local parts2 = split_n(parts[2], /,/, F, 4);
+		if ( |parts2| > 1 )
+			classification_map[|classification_map|+1] = parts2[1];
+		}
+	}
+
 event bro_init()
 	{
+	Log::create_stream(Unified2::LOG, [$columns=Info, $ev=log_unified2]);
+
+	if ( sid_msg != "" )
+		{
+		Input::add_event([$source=sid_msg,
+		                  $reader=Input::READER_RAW,
+		                  $mode=Input::REREAD,
+		                  $name=sid_msg,
+		                  $fields=Unified2::OneLine,
+		                  $want_record=F,
+		                  $ev=Unified2::read_sid_msg_line]);
+		}
+
+	if ( gen_msg != "" )
+		{
+		Input::add_event([$source=gen_msg,
+		                  $name=gen_msg,
+		                  $reader=Input::READER_RAW,
+		                  $mode=Input::REREAD,
+		                  $fields=Unified2::OneLine,
+		                  $want_record=F,
+		                  $ev=Unified2::read_gen_msg_line]);
+		}
+
+	if ( classification_config != "" )
+		{
+		Input::add_event([$source=classification_config,
+		                  $name=classification_config,
+		                  $reader=Input::READER_RAW,
+		                  $mode=Input::REREAD,
+		                  $fields=Unified2::OneLine,
+		                  $want_record=F,
+		                  $ev=Unified2::read_classification_line]);
+		}
+
 	if ( watch_dir != "" )
 		{
 		Dir::monitor(watch_dir, function(fname: string)
@@ -61,20 +178,45 @@ event file_new(f: fa_file)
 		compress_path(watch_dir) == file_dir )
 		{
 		Files::add_analyzer(f, Files::ANALYZER_UNIFIED2);
-		f$unified2 = Info();
+		f$u2_events = table();
 		}
 	}
 
 event unified2_event(f: fa_file, ev: Unified2::IDSEvent)
 	{
-	f$unified2$current_event = ev;
+	f$u2_events[ev$event_id] = ev;
 	}
 
 event unified2_packet(f: fa_file, pkt: Unified2::Packet)
 	{
-	if ( f$unified2?$current_event )
+	if ( f?$u2_events && pkt$event_id in f$u2_events)
 		{
-		event Unified2::alert(f, f$unified2$current_event, pkt);
+		local ev = f$u2_events[pkt$event_id];
-		delete f$unified2$current_event;
+		event Unified2::alert(f, ev, pkt);
+		delete f$u2_events[pkt$event_id];
 		}
 	}
+
+event Unified2::alert(f: fa_file, ev: IDSEvent, pkt: Packet)
+	{
+	local info = Info($ts=ev$ts, 
+	                  $id=PacketID($src_ip=ev$src_ip, $src_p=ev$src_p,
+	                               $dst_ip=ev$dst_ip, $dst_p=ev$dst_p),
+	                  $sensor_id=ev$sensor_id,
+	                  $signature_id=ev$signature_id,
+	                  $generator_id=ev$generator_id,
+	                  $signature_revision=ev$signature_revision,
+	                  $classification_id=ev$classification_id,
+	                  $priority_id=ev$priority_id,
+	                  $event_id=ev$event_id,
+	                  $packet=pkt$data);
+
+	if ( ev$signature_id in sid_map )
+		info$signature=sid_map[ev$signature_id];
+	if ( ev$generator_id in gen_map )
+		info$generator=gen_map[ev$generator_id];
+	if ( ev$classification_id in classification_map )
+		info$classification=classification_map[ev$classification_id];
+
+	Log::write(LOG, info);
+	}

scripts/base/init-bare.bro

@@ -2713,12 +2713,12 @@ export {
 	};
 
 	type Unified2::Packet: record {
-		sensor_id:          count;
+		sensor_id:    count;
-		event_id:           count;
+		event_id:     count;
-		event_second:       count;
+		event_second: count;
-		packet_ts:          time;
+		packet_ts:    time;
-		link_type:          count;
+		link_type:    count;
-		packet_data:        string;
+		data:         string;
 	};
 }
 

src/file_analysis/analyzer/unified2/unified2-analyzer.pac

@@ -61,7 +61,7 @@ refine flow Flow += {
 	#	return true;
 	#	%}
 
-	function proc_legacy_ids_event(ev: LegacyIDSEvent) : bool
+	function proc_ids_event(ev: IDS_Event) : bool
 		%{
 		if ( ::unified2_event )
 			{
@@ -88,7 +88,7 @@ refine flow Flow += {
 		return true;
 		%}
 
-	function proc_ids_event(ev: IDSEvent) : bool
+	function proc_ids_event_2(ev: IDS_Event_2) : bool
 		%{
 		if ( ::unified2_event )
 			{
@@ -152,12 +152,12 @@ refine flow Flow += {
 #	proc : bool = $context.flow.proc_record(this);
 #};
 
-refine typeattr LegacyIDSEvent += &let {
+refine typeattr IDS_Event += &let {
-	proc : bool = $context.flow.proc_legacy_ids_event(this);
+	proc : bool = $context.flow.proc_ids_event(this);
 };
 
-refine typeattr IDSEvent += &let {
+refine typeattr IDS_Event_2 += &let {
-	proc : bool = $context.flow.proc_ids_event(this);
+	proc : bool = $context.flow.proc_ids_event_2(this);
 };
 
 refine typeattr Packet += &let {

src/file_analysis/analyzer/unified2/unified2-file.pac

@@ -18,16 +18,16 @@ type Record = record {
 	length:  uint32;
 	data:   case rtype of {
 		PACKET            -> packet:              Packet(this);
-		IDS_EVENT         -> ids_event:           LegacyIDSEvent(this, 1);
+		IDS_EVENT         -> ids_event:           IDS_Event(this, 1);
-		IDS_EVENT_IPV6    -> ids_event_ipv6:      LegacyIDSEvent(this, 4);
+		IDS_EVENT_IPV6    -> ids_event_ipv6:      IDS_Event(this, 4);
-		IDS_EVENT_2       -> ids_event_vlan:      IDSEvent(this, 1);
+		IDS_EVENT_2       -> ids_event_vlan:      IDS_Event_2(this, 1);
-		IDS_EVENT_IPV6_2  -> ids_event_ipv6_vlan: IDSEvent(this, 4);
+		IDS_EVENT_IPV6_2  -> ids_event_ipv6_vlan: IDS_Event_2(this, 4);
 		#EXTRA_DATA        -> extra_data:          ExtraData(this);
 		default           -> unknown_record_type: UnknownRecordType(this);
 	};
 } &byteorder=bigendian &length=length+8;
 
-type LegacyIDSEvent(rec: Record, ip_len: int) = record {
+type IDS_Event(rec: Record, ip_len: int) = record {
 	sensor_id:          uint32;
 	event_id:           uint32;
 	ts:                 Time;
@@ -44,7 +44,7 @@ type LegacyIDSEvent(rec: Record, ip_len: int) = record {
 	packet_action:      uint8;
 } &byteorder=bigendian;
 
-type IDSEvent(rec: Record, ip_len: int) = record {
+type IDS_Event_2(rec: Record, ip_len: int) = record {
 	sensor_id:          uint32;
 	event_id:           uint32;
 	ts:                 Time;

testing/btest/Baseline/scripts.base.files.unified2.alert/unified2.log

@@ -0,0 +1,11 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	unified2
+#open	2013-08-13-07-16-01
+#fields	ts	id.src_ip	id.src_p	id.dst_ip	id.dst_p	sensor_id	signature_id	signature	generator_id	generator	signature_revision	classification_id	classification	priority_id	event_id	packet
+#types	time	addr	port	addr	port	count	count	string	count	string	count	count	string	count	count	string
+1323827323.000000	192.168.1.72	50185	74.125.225.49	80	0	2003058	ET MALWARE 180solutions (Zango) Spyware Installer Download	1	snort general alert	5	21	trojan-activity	1	2	\xd80bH\xc5\xb5x\xca9\xb7\xe4r\x08\x00E\x10\x00\\x1a\xce@\x00@\x062\x1f\xc0\xa8\x01HJ}\xe11\xc4\x09\x00P*\xa8bv]z/\xde\x80\x18\x82+\x88,\x00\x00\x01\x01\x08\x0a\x17J\x83Q\xfe\xad\xac\x1aGET /Zango/ZangoInstaller.exe HTTP/1.0\x0d\x0a
+1323827344.000000	192.168.1.72	49862	199.47.216.144	80	0	2012647	ET POLICY Dropbox.com Offsite File Backup in Use	1	snort general alert	3	33	policy-violation	1	3	\xd80bH\xc5\xb5x\xca9\xb7\xe4r\x08\x00E\x00\x00\xf8Q\xdf@\x00@\x06\x86p\xc0\xa8\x01H\xc7/\xd8\x90\xc2\xc6\x00P\x9cm\x97U\xf07\x084\x80\x18\x82\x18%<\x00\x00\x01\x01\x08\x0a\x17J\xd7\xde\x00\x92\x81\xc5GET /subscribe?host_int=43112345&ns_map=123456_1234524412104916591&ts=1323827344 HTTP/1.1\x0d\x0aHost: notify1.dropbox.com\x0d\x0aAccept-Encoding: identity\x0d\x0aConnection: keep-alive\x0d\x0aX-Dropbox-Locale: en_US\x0d\x0a\x0d\x0a
+#close	2013-08-13-07-16-01

testing/btest/btest.cfg

@@ -14,6 +14,7 @@ LC_ALL=C
 BTEST_PATH=%(testbase)s/../../aux/btest
 PATH=%(testbase)s/../../build/src:%(testbase)s/../scripts:%(testbase)s/../../aux/btest:%(testbase)s/../../aux/bro-aux/bro-cut:%(default_path)s
 TRACES=%(testbase)s/Traces
+FILES=%(testbase)s/Files
 SCRIPTS=%(testbase)s/../scripts
 DIST=%(testbase)s/../..
 BUILD=%(testbase)s/../../build

testing/btest/scripts/base/files/unified2/alert.bro

@@ -0,0 +1,76 @@
+# @TEST-EXEC: bro -b %INPUT Unified2::watch_file=$FILES/unified2.u2
+# @TEST-EXEC: btest-diff unified2.log
+
+@TEST-START-FILE sid_msg.map
+2003058 || ET MALWARE 180solutions (Zango) Spyware Installer Download || url,doc.emergingthreats.net/bin/view/Main/2003058 || url,securityresponse.symantec.com/avcenter/venc/data/pf/adware.180search.html
+2012647 || ET POLICY Dropbox.com Offsite File Backup in Use || url,dereknewton.com/2011/04/dropbox-authentication-static-host-ids/ || url,www.dropbox.com
+@TEST-END-FILE
+
+@TEST-START-FILE gen_msg.map
+1 || 1 || snort general alert
+2 || 1 || tag: Tagged Packet
+3 || 1 || snort dynamic alert
+100 || 1 || spp_portscan: Portscan Detected
+100 || 2 || spp_portscan: Portscan Status
+100 || 3 || spp_portscan: Portscan Ended
+101 || 1 || spp_minfrag: minfrag alert
+@TEST-END-FILE
+
+@TEST-START-FILE classification.config
+#
+# config classification:shortname,short description,priority
+#
+
+#Traditional classifications. These will be replaced soon
+
+config classification: not-suspicious,Not Suspicious Traffic,3
+config classification: unknown,Unknown Traffic,3
+config classification: bad-unknown,Potentially Bad Traffic, 2
+config classification: attempted-recon,Attempted Information Leak,2
+config classification: successful-recon-limited,Information Leak,2
+config classification: successful-recon-largescale,Large Scale Information Leak,2
+config classification: attempted-dos,Attempted Denial of Service,2
+config classification: successful-dos,Denial of Service,2
+config classification: attempted-user,Attempted User Privilege Gain,1
+config classification: unsuccessful-user,Unsuccessful User Privilege Gain,1
+config classification: successful-user,Successful User Privilege Gain,1
+config classification: attempted-admin,Attempted Administrator Privilege Gain,1
+config classification: successful-admin,Successful Administrator Privilege Gain,1
+config classification: rpc-portmap-decode,Decode of an RPC Query,2
+config classification: shellcode-detect,Executable Code was Detected,1
+config classification: string-detect,A Suspicious String was Detected,3
+config classification: suspicious-filename-detect,A Suspicious Filename was Detected,2
+config classification: suspicious-login,An Attempted Login Using a Suspicious Username was Detected,2
+config classification: system-call-detect,A System Call was Detected,2
+config classification: tcp-connection,A TCP Connection was Detected,4
+config classification: trojan-activity,A Network Trojan was Detected, 1
+config classification: unusual-client-port-connection,A Client was Using an Unusual Port,2
+config classification: network-scan,Detection of a Network Scan,3
+config classification: denial-of-service,Detection of a Denial of Service Attack,2
+config classification: non-standard-protocol,Detection of a Non-Standard Protocol or Event,2
+config classification: protocol-command-decode,Generic Protocol Command Decode,3
+config classification: web-application-activity,Access to a Potentially Vulnerable Web Application,2
+config classification: web-application-attack,Web Application Attack,1
+config classification: misc-activity,Misc activity,3
+config classification: misc-attack,Misc Attack,2
+config classification: icmp-event,Generic ICMP event,3
+config classification: inappropriate-content,Inappropriate Content was Detected,1
+config classification: policy-violation,Potential Corporate Privacy Violation,1
+config classification: default-login-attempt,Attempt to Login By a Default Username and Password,2
+@TEST-END-FILE
+
+redef exit_only_after_terminate = T;
+
+@load base/files/unified2
+
+redef Unified2::sid_msg = @DIR+"/sid_msg.map";
+redef Unified2::gen_msg = @DIR+"/gen_msg.map";
+redef Unified2::classification_config = @DIR+"/classification.config";
+global i = 0;
+
+event Unified2::alert(f: fa_file, ev: Unified2::IDSEvent, pkt: Unified2::Packet)
+	{
+	++i;
+	if ( i == 2 )
+		terminate();
+	}