Working unified2 analyzer.

- No output by default yet.  Most of the activity is centered
   around generating the Unified2::alert event which ties together
   an IDSEvent and a packet.

scripts/base/files/unified2/main.bro

@@ -1,16 +1,71 @@
 
+@load base/utils/dir
+@load base/utils/paths
 
+module Unified2;
+
+export {
+	## Directory to watch for Unified2 files.
+	const watch_file = "" &redef;
+
+	## File to watch for Unified2 records.
+	const watch_dir = "" &redef;
+
+	## Reconstructed "alert" which combines related events
+	## and packets.
+	global alert: event(f: fa_file, ev: Unified2::IDSEvent, pkt: Unified2::Packet);
+
+	type Info: record {
+		current_event: Unified2::IDSEvent &optional;
+	};
+
+	redef record fa_file += {
+		unified2: Info &optional;
+	};
+}
+
+event bro_init()
+	{
+	if ( watch_dir != "" )
+		{
+		Dir::monitor(watch_dir, function(fname: string)
+			{
+			Input::add_analysis([$source=fname, 
+			                     $reader=Input::READER_BINARY,
+			                     $mode=Input::MANUAL, 
+			                     $name=fname]);
+			}, 10secs);
+		}
+
+	if ( watch_file != "" )
+		{
+		Input::add_analysis([$source=watch_file, 
+		                     $reader=Input::READER_BINARY,
+		                     $mode=Input::MANUAL, 
+		                     $name=watch_file]);
+		}
+	}
 
 event file_new(f: fa_file)
 	{
-	print "found a file";
-	print f$mime_type;
-	print Files::add_analyzer(f, Files::ANALYZER_UNIFIED2);
-	}
-
-event unified2_alert(f: fa_file, alert: count)
+	if ( f$source in watch_file || 
+		compress_path(extract_path(f$source)) == compress_path(watch_dir) )
 		{
-	print "yaayyaya!!!";
-
-	print alert;
+		Files::add_analyzer(f, Files::ANALYZER_UNIFIED2);
+		f$unified2 = Info();
+		}
+	}
+
+event unified2_event(f: fa_file, ev: Unified2::IDSEvent)
+	{
+	f$unified2$current_event = ev;
+	}
+
+event unified2_packet(f: fa_file, pkt: Unified2::Packet)
+	{
+	if ( f$unified2?$current_event )
+		{
+		event Unified2::alert(f, f$unified2$current_event, pkt);
+		delete f$unified2$current_event;
+		}
 	}

scripts/base/init-bare.bro

@@ -2686,6 +2686,39 @@ type ModbusHeaders: record {
 	function_code: count;
 };
 
+module Unified2;
+export {
+	type Unified2::IDSEvent: record {
+		sensor_id:          count;
+		event_id:           count;
+		ts:                 time;
+		signature_id:       count;
+		generator_id:       count;
+		signature_revision: count;
+		classification_id:  count;
+		priority_id:        count;
+		src_ip:             addr;
+		dst_ip:             addr;
+		src_p:              port;
+		dst_p:              port;
+		impact_flag:        count;
+		impact:             count;
+		blocked:            count;
+		mpls_label:         count  &optional; ## Not available in "legacy" IDS events.
+		vlan_id:            count  &optional; ## Not available in "legacy" IDS events.
+		packet_action:      count  &optional; ## Only available in "legacy" IDS events.
+	};
+
+	type Unified2::Packet: record {
+		sensor_id:          count;
+		event_id:           count;
+		event_second:       count;
+		packet_ts:          time;
+		link_type:          count;
+		packet_data:        string;
+	};
+}
+
 module SOCKS;
 export {
 	## This record is for a SOCKS client or server to provide either a

scripts/base/utils/paths.bro

@@ -9,13 +9,12 @@ const absolute_path_pat = /(\/|[A-Za-z]:[\\\/]).*/;
 ## Returns: the first absolute path found in input string, else an empty string
 function extract_path(input: string): string
 	{
-	const dir_pattern = /(\/|[A-Za-z]:[\\\/])([^\"\ ]|(\\\ ))*/;
+	local dir_pattern = /([^\\\/]+)$/;
 	local parts = split_all(input, dir_pattern);
-
 	if ( |parts| < 3 )
 		return "";
 
-	return parts[2];
+	return parts[1];
 	}
 
 ## Compresses a given path by removing '..'s and the parent directory it

src/file_analysis/analyzer/unified2/CMakeLists.txt

@@ -5,6 +5,6 @@ include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}
 
 bro_plugin_begin(Bro Unified2)
 bro_plugin_cc(Unified2.cc Plugin.cc ../../Analyzer.cc)
-bro_plugin_bif(events.bif)
+bro_plugin_bif(events.bif types.bif)
 bro_plugin_pac(unified2.pac unified2-file.pac unified2-analyzer.pac)
 bro_plugin_end()

src/file_analysis/analyzer/unified2/Plugin.cc

@@ -1,4 +1,5 @@
 #include "plugin/Plugin.h"
+
 #include "file_analysis/Component.h"
 
 #include "Unified2.h"
@@ -21,6 +22,9 @@ protected:
 
 		extern std::list<std::pair<const char*, int> > __bif_events_init();
 		AddBifInitFunction(&__bif_events_init);
+
+		extern std::list<std::pair<const char*, int> > __bif_types_init();
+		AddBifInitFunction(&__bif_types_init);
 		}
 };
 

src/file_analysis/analyzer/unified2/Unified2.cc

@@ -1,7 +1,5 @@
 // See the file "COPYING" in the main distribution directory for copyright.
 
-#include <string>
-
 #include "Unified2.h"
 #include "file_analysis/Manager.h"
 
@@ -15,6 +13,7 @@ Unified2::Unified2(RecordVal* args, File* file)
 
 Unified2::~Unified2()
 	{
+	delete interp;
 	}
 
 file_analysis::Analyzer* Unified2::Instantiate(RecordVal* args, File* file)
@@ -23,7 +22,16 @@ file_analysis::Analyzer* Unified2::Instantiate(RecordVal* args, File* file)
 	}
 
 bool Unified2::DeliverStream(const u_char* data, uint64 len)
+	{
+	try
 		{
 		interp->NewData(true, data, data + len);
+		}
+	catch ( const binpac::Exception& e )
+		{
+		printf("Binpac exception: %s\n", e.c_msg());
+		return false;
+		}
+
 	return true;
 	}

src/file_analysis/analyzer/unified2/events.bif

@@ -1,2 +1,7 @@
 
-event unified2_alert%(f: fa_file, alert: count%);
+## Abstract all of the various Unified2 event formats into 
+## a single event.
+event unified2_event%(f: fa_file, ev: Unified2::IDSEvent%);
+
+## The Unified2 packet format event.
+event unified2_packet%(f: fa_file, pkt: Unified2::Packet%);

src/file_analysis/analyzer/unified2/types.bif

@@ -1 +1,2 @@
-type Unified2Alert: record;
+type Unified2::IDSEvent: record;
+type Unified2::Packet: record;

src/file_analysis/analyzer/unified2/unified2-analyzer.pac

@@ -2,6 +2,8 @@
 #include "Event.h"
 #include "file_analysis/File.h"
 #include "events.bif.h"
+#include "types.bif.h"
+#include "IPAddr.h"
 %}
 
 refine flow Flow += {
@@ -18,18 +20,150 @@ refine flow Flow += {
 	%cleanup{
 	%}
 
-	function proc_ids_event(ev: IDSEvent) : bool
+	function ts_to_double(ts: Time): double
 		%{
+		double t = ${ts.seconds} + (${ts.microseconds} / 1000000);
+		return t;
+		%}
+
+	function unified2_addr_to_bro_addr(a: uint32[]): AddrVal
+		%{
+		if ( a->size() == 1 )
+			{
+			return new AddrVal(IPAddr(IPv4, &(a->at(0)), IPAddr::Host));
+			}
+		else if ( a->size() == 4 )
+			{
+			uint32 tmp[4] = { a->at(0), a->at(1), a->at(2), a->at(3) };
+			return new AddrVal(IPAddr(IPv6, tmp, IPAddr::Host));
+			}
+		else
+			{
+			// Should never reach here.
+			return new AddrVal(1);
+			}
+		%}
+
+	function to_port(n: uint16, p: uint8): PortVal
+		%{
+		TransportProto proto = TRANSPORT_UNKNOWN;
+		switch ( p ) {
+		case 1: proto = TRANSPORT_ICMP; break;
+		case 6: proto = TRANSPORT_TCP; break;
+		case 17: proto = TRANSPORT_UDP; break;
+		}
+
+		return new PortVal(n, proto);
+		%}
+
+	#function proc_record(rec: Record) : bool
+	#	%{
+	#	return true;
+	#	%}
+
+	function proc_legacy_ids_event(ev: LegacyIDSEvent) : bool
+		%{
+		if ( ::unified2_event )
+			{
+			RecordVal* ids_event = new RecordVal(BifType::Record::Unified2::IDSEvent);
+			ids_event->Assign(0, new Val(${ev.sensor_id}, TYPE_COUNT));
+			ids_event->Assign(1, new Val(${ev.event_id}, TYPE_COUNT));
+			ids_event->Assign(2, new Val(ts_to_double(${ev.ts}), TYPE_TIME));
+			ids_event->Assign(3, new Val(${ev.signature_id}, TYPE_COUNT));
+			ids_event->Assign(4, new Val(${ev.generator_id}, TYPE_COUNT));
+			ids_event->Assign(5, new Val(${ev.signature_revision}, TYPE_COUNT));
+			ids_event->Assign(6, new Val(${ev.classification_id}, TYPE_COUNT));
+			ids_event->Assign(7, new Val(${ev.priority_id}, TYPE_COUNT));
+			ids_event->Assign(8, unified2_addr_to_bro_addr(${ev.src_ip}));
+			ids_event->Assign(9, unified2_addr_to_bro_addr(${ev.dst_ip}));
+			ids_event->Assign(10, to_port(${ev.src_p}, ${ev.protocol}));
+			ids_event->Assign(11, to_port(${ev.dst_p}, ${ev.protocol}));
+			ids_event->Assign(17, new Val(${ev.packet_action}, TYPE_COUNT));
+
 			val_list* vl = new val_list();
 			vl->append(connection()->bro_analyzer()->GetFile()->GetVal()->Ref());
-		vl->append(new Val(${ev.signature_id}, TYPE_COUNT));
-		mgr.QueueEvent(::unified2_alert, vl, SOURCE_LOCAL);
+			vl->append(ids_event);
+			mgr.QueueEvent(::unified2_event, vl, SOURCE_LOCAL);
+			}
+		return true;
+		%}
+
+	function proc_ids_event(ev: IDSEvent) : bool
+		%{
+		if ( ::unified2_event )
+			{
+			RecordVal* ids_event = new RecordVal(BifType::Record::Unified2::IDSEvent);
+			ids_event->Assign(0, new Val(${ev.sensor_id}, TYPE_COUNT));
+			ids_event->Assign(1, new Val(${ev.event_id}, TYPE_COUNT));
+			ids_event->Assign(2, new Val(ts_to_double(${ev.ts}), TYPE_TIME));
+			ids_event->Assign(3, new Val(${ev.signature_id}, TYPE_COUNT));
+			ids_event->Assign(4, new Val(${ev.generator_id}, TYPE_COUNT));
+			ids_event->Assign(5, new Val(${ev.signature_revision}, TYPE_COUNT));
+			ids_event->Assign(6, new Val(${ev.classification_id}, TYPE_COUNT));
+			ids_event->Assign(7, new Val(${ev.priority_id}, TYPE_COUNT));
+			ids_event->Assign(8, unified2_addr_to_bro_addr(${ev.src_ip}));
+			ids_event->Assign(9, unified2_addr_to_bro_addr(${ev.dst_ip}));
+			ids_event->Assign(10, to_port(${ev.src_p}, ${ev.protocol}));
+			ids_event->Assign(11, to_port(${ev.dst_p}, ${ev.protocol}));
+			ids_event->Assign(12, new Val(${ev.impact_flag}, TYPE_COUNT));
+			ids_event->Assign(13, new Val(${ev.impact}, TYPE_COUNT));
+			ids_event->Assign(14, new Val(${ev.blocked}, TYPE_COUNT));
+			ids_event->Assign(15, new Val(${ev.mpls_label}, TYPE_COUNT));
+			ids_event->Assign(16, new Val(${ev.vlan_id}, TYPE_COUNT));
+			
+			val_list* vl = new val_list();
+			vl->append(connection()->bro_analyzer()->GetFile()->GetVal()->Ref());
+			vl->append(ids_event);
+			mgr.QueueEvent(::unified2_event, vl, SOURCE_LOCAL);
+			}
 
 		return true;
 		%}
+
+	function proc_packet(pkt: Packet) : bool
+		%{
+		if ( ::unified2_packet )
+			{
+			RecordVal* packet = new RecordVal(BifType::Record::Unified2::Packet);
+			packet->Assign(0, new Val(${pkt.sensor_id}, TYPE_COUNT));
+			packet->Assign(1, new Val(${pkt.event_id}, TYPE_COUNT));
+			packet->Assign(2, new Val(${pkt.event_second}, TYPE_COUNT));
+			packet->Assign(3, new Val(ts_to_double(${pkt.packet_ts}), TYPE_TIME));
+			packet->Assign(4, new Val(${pkt.link_type}, TYPE_COUNT));
+			packet->Assign(5, bytestring_to_val(${pkt.packet_data}));
+			
+			val_list* vl = new val_list();
+			vl->append(connection()->bro_analyzer()->GetFile()->GetVal()->Ref());
+			vl->append(packet);
+			mgr.QueueEvent(::unified2_packet, vl, SOURCE_LOCAL);
+			}
+
+		return true;
+		%}
+
+	#function proc_unknown_record_type(rec: UnknownRecordType) : bool
+	#	%{
+	#	printf("unknown packet type\n");
+	#	return true;
+	#	%}
 };
 
+#refine typeattr Record += &let {
+#	proc : bool = $context.flow.proc_record(this);
+#};
+
+refine typeattr LegacyIDSEvent += &let {
+	proc : bool = $context.flow.proc_legacy_ids_event(this);
+};
 
 refine typeattr IDSEvent += &let {
 	proc : bool = $context.flow.proc_ids_event(this);
 };
+
+refine typeattr Packet += &let {
+	proc : bool = $context.flow.proc_packet(this);
+};
+
+#refine typeattr UnknownRecordType += &let {
+#	proc : bool = $context.flow.proc_unknown_record_type(this);
+#};

src/file_analysis/analyzer/unified2/unified2-file.pac

@@ -1,13 +1,10 @@
 
 enum Types {
-	EVENT               = 0,
-	PACKET              = 1,
-	IDS_EVENT           = 2,
+	PACKET           = 2,
+	IDS_EVENT        = 7,
 	IDS_EVENT_IPV6   = 72,
-	IDS_EVENT_MPLS      = 99,
-	IDS_EVENT_IPV6_MPLS = 100,
-	IDS_EVENT_VLAN      = 104,
-	IDS_EVENT_IPV6_VLAN = 105,
+	IDS_EVENT_2      = 104,
+	IDS_EVENT_IPV6_2 = 105,
 	EXTRA_DATA       = 110,
 };
 
@@ -16,40 +13,21 @@ type Time = record {
 	microseconds: uint32;
 } &byteorder=bigendian;
 
-type v4Addr = record {
-	u1: uint32;
-};
-
-type v6Addr = record {
-	u1: uint32;
-	u2: uint32;
-	u3: uint32;
-	u4: uint32;
-};
-
-type Addr(ip_ver: int) = case ip_ver of {
-	4 -> v4: v4Addr;
-	6 -> v6: v6Addr;
-} &byteorder=bigendian;
-
 type Record = record {
 	rtype:   uint32;
 	length:  uint32;
 	data:   case rtype of {
-		# EVENT               -> event:               Event(this);
 		PACKET            -> packet:              Packet(this);
-		IDS_EVENT           -> ids_event:           LegacyIDSEvent(this, 4);
-		IDS_EVENT_IPV6      -> ids_event_ipv6:      LegacyIDSEvent(this, 6);
-		# IDS_EVENT_MPLS      -> ids_event_mpls:      IDSEvent(this, 4);
-		# IDS_EVENT_IPV6_MPLS -> ids_event_ipv6_mpls: IDSEvent(this, 6);
-		IDS_EVENT_VLAN      -> ids_event_vlan:      IDSEvent(this, 4);
-		IDS_EVENT_IPV6_VLAN -> ids_event_ipv6_vlan: IDSEvent(this, 6);
-		EXTRA_DATA          -> extra_data:          ExtraData(this);
+		IDS_EVENT         -> ids_event:           LegacyIDSEvent(this, 1);
+		IDS_EVENT_IPV6    -> ids_event_ipv6:      LegacyIDSEvent(this, 4);
+		IDS_EVENT_2       -> ids_event_vlan:      IDSEvent(this, 1);
+		IDS_EVENT_IPV6_2  -> ids_event_ipv6_vlan: IDSEvent(this, 4);
+		#EXTRA_DATA        -> extra_data:          ExtraData(this);
 		default           -> unknown_record_type: UnknownRecordType(this);
 	};
 } &byteorder=bigendian &length=length+8;
 
-type LegacyIDSEvent(rec: Record, ip_ver: int) = record {
+type LegacyIDSEvent(rec: Record, ip_len: int) = record {
 	sensor_id:          uint32;
 	event_id:           uint32;
 	ts:                 Time;
@@ -58,15 +36,15 @@ type LegacyIDSEvent(rec: Record, ip_ver: int) = record {
 	signature_revision: uint32;
 	classification_id:  uint32;
 	priority_id:        uint32;
-	src_ip:             Addr(ip_ver);
-	dst_ip:             Addr(ip_ver);
+	src_ip:             uint32[ip_len];
+	dst_ip:             uint32[ip_len];
 	src_p:              uint16;
 	dst_p:              uint16;
 	protocol:           uint8;
 	packet_action:      uint8;
-};
+} &byteorder=bigendian;
 
-type IDSEvent(rec: Record, ip_ver: int) = record {
+type IDSEvent(rec: Record, ip_len: int) = record {
 	sensor_id:          uint32;
 	event_id:           uint32;
 	ts:                 Time;
@@ -75,8 +53,8 @@ type IDSEvent(rec: Record, ip_ver: int) = record {
 	signature_revision: uint32;
 	classification_id:  uint32;
 	priority_id:        uint32;
-	src_ip:             Addr(ip_ver);
-	dst_ip:             Addr(ip_ver);
+	src_ip:             uint32[ip_len];
+	dst_ip:             uint32[ip_len];
 	src_p:              uint16;
 	dst_p:              uint16;
 	protocol:           uint8;
@@ -85,7 +63,7 @@ type IDSEvent(rec: Record, ip_ver: int) = record {
 	blocked:            uint8;
 	mpls_label:         uint32;
 	vlan_id:            uint16;
-	:                   uint16;
+	pad:                uint16;
 } &byteorder=bigendian;
 
 type Packet(rec: Record) = record {
@@ -96,7 +74,7 @@ type Packet(rec: Record) = record {
 	link_type:          uint32;
 	packet_len:         uint32;
 	packet_data:        bytestring &length=packet_len;
-} &byteorder=bigendian &length=rec.length;
+} &byteorder=bigendian;
 
 type ExtraData(rec: Record) = record {
 	sensor_id:          uint32;
@@ -111,7 +89,3 @@ type ExtraData(rec: Record) = record {
 type UnknownRecordType(rec: Record) = record {
 	data: bytestring &transient &length=rec.length;
 } &byteorder=bigendian &length=rec.length;
-
-type File = record {
-	alerts:  Record[] &transient &until($element <= 0);
-} &byteorder=bigendian;

src/file_analysis/analyzer/unified2/unified2.pac

@@ -15,7 +15,7 @@ analyzer Unified2_Analyzer(bro_analyzer: BroFileAnalyzer) {
 %include unified2-file.pac
 
 flow Flow {
-	flowunit = File withcontext(connection, this);
+	flowunit = Record withcontext(connection, this);
 };
 
 %include unified2-analyzer.pac