Merge remote-tracking branch 'origin/topic/seth/unified2-analyzer'

BIT-1054 #merged

* origin/topic/seth/unified2-analyzer:
  Fixes in case a packet isn't seen that matches an event.
  Finished work on unified2 analyzer.
  Fixed some tests.
  Working unified2 analyzer.
  Unified2 file analyzer updated to new plugin style.
  Adding the unified2 analyzer.

Conflicts:
	testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log
	testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log

src/file_analysis/analyzer/CMakeLists.txt

@@ -1,3 +1,4 @@
 add_subdirectory(data_event)
 add_subdirectory(extract)
 add_subdirectory(hash)
+add_subdirectory(unified2)

src/file_analysis/analyzer/unified2/CMakeLists.txt

@@ -0,0 +1,11 @@
+
+include(BroPlugin)
+
+include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}
+                           ${CMAKE_CURRENT_BINARY_DIR})
+
+bro_plugin_begin(Bro Unified2)
+bro_plugin_cc(Unified2.cc Plugin.cc ../../Analyzer.cc)
+bro_plugin_bif(events.bif types.bif)
+bro_plugin_pac(unified2.pac unified2-file.pac unified2-analyzer.pac)
+bro_plugin_end()

src/file_analysis/analyzer/unified2/Plugin.cc

@@ -0,0 +1,35 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "plugin/Plugin.h"
+
+#include "file_analysis/Component.h"
+
+#include "Unified2.h"
+
+namespace plugin { namespace Bro_Unified2 {
+
+class Plugin : public plugin::Plugin {
+protected:
+	void InitPreScript()
+		{
+		SetName("Bro::Unified2");
+		SetVersion(-1);
+		SetAPIVersion(BRO_PLUGIN_API_VERSION);
+		SetDynamicPlugin(false);
+
+		SetDescription("Analyze Unified2 alert files.");
+
+		AddComponent(new ::file_analysis::Component("UNIFIED2",
+		        ::file_analysis::Unified2::Instantiate));
+
+		extern std::list<std::pair<const char*, int> > __bif_events_init();
+		AddBifInitFunction(&__bif_events_init);
+
+		extern std::list<std::pair<const char*, int> > __bif_types_init();
+		AddBifInitFunction(&__bif_types_init);
+		}
+};
+
+Plugin __plugin;
+
+} }

src/file_analysis/analyzer/unified2/Unified2.cc

@@ -0,0 +1,38 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "Unified2.h"
+#include "file_analysis/Manager.h"
+
+using namespace file_analysis;
+
+Unified2::Unified2(RecordVal* args, File* file)
+    : file_analysis::Analyzer(file_mgr->GetComponentTag("UNIFIED2"), args, file)
+	{
+	interp = new binpac::Unified2::Unified2_Analyzer(this);
+	}
+
+Unified2::~Unified2()
+	{
+	delete interp;
+	}
+
+file_analysis::Analyzer* Unified2::Instantiate(RecordVal* args, File* file)
+	{
+	return new Unified2(args, file);
+	}
+
+bool Unified2::DeliverStream(const u_char* data, uint64 len)
+	{
+	try
+		{
+		interp->NewData(true, data, data + len);
+		}
+
+	catch ( const binpac::Exception& e )
+		{
+		printf("Binpac exception: %s\n", e.c_msg());
+		return false;
+		}
+
+	return true;
+	}

src/file_analysis/analyzer/unified2/Unified2.h

@@ -0,0 +1,38 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#ifndef FILE_ANALYSIS_UNIFIED2_H
+#define FILE_ANALYSIS_UNIFIED2_H
+
+#include <string>
+
+#include "Val.h"
+#include "File.h"
+#include "Analyzer.h"
+#include "unified2_pac.h"
+
+namespace file_analysis {
+
+/**
+ * An analyzer to extract content of files from local disk.
+ */
+class Unified2 : public file_analysis::Analyzer {
+public:
+	virtual ~Unified2();
+
+	virtual bool DeliverStream(const u_char* data, uint64 len);
+
+	static file_analysis::Analyzer* Instantiate(RecordVal* args, File* file);
+
+protected:
+	Unified2(RecordVal* args, File* file);
+
+private:
+	binpac::Unified2::Unified2_Analyzer* interp;
+
+	string filename;
+	int fd;
+};
+
+} // namespace file_analysis
+
+#endif

src/file_analysis/analyzer/unified2/events.bif

@@ -0,0 +1,7 @@
+
+## Abstract all of the various Unified2 event formats into 
+## a single event.
+event unified2_event%(f: fa_file, ev: Unified2::IDSEvent%);
+
+## The Unified2 packet format event.
+event unified2_packet%(f: fa_file, pkt: Unified2::Packet%);

src/file_analysis/analyzer/unified2/types.bif

@@ -0,0 +1,2 @@
+type Unified2::IDSEvent: record;
+type Unified2::Packet: record;

src/file_analysis/analyzer/unified2/unified2-analyzer.pac

@@ -0,0 +1,170 @@
+
+%extern{
+#include "Event.h"
+#include "file_analysis/File.h"
+#include "events.bif.h"
+#include "types.bif.h"
+#include "IPAddr.h"
+%}
+
+refine flow Flow += {
+
+	%member{
+	%}
+
+	%init{
+	%}
+
+	%eof{
+	%}
+
+	%cleanup{
+	%}
+
+	function ts_to_double(ts: Time): double
+		%{
+		double t = ${ts.seconds} + (${ts.microseconds} / 1000000);
+		return t;
+		%}
+
+	function unified2_addr_to_bro_addr(a: uint32[]): AddrVal
+		%{
+		if ( a->size() == 1 )
+			{
+			return new AddrVal(IPAddr(IPv4, &(a->at(0)), IPAddr::Host));
+			}
+		else if ( a->size() == 4 )
+			{
+			uint32 tmp[4] = { a->at(0), a->at(1), a->at(2), a->at(3) };
+			return new AddrVal(IPAddr(IPv6, tmp, IPAddr::Host));
+			}
+		else
+			{
+			// Should never reach here.
+			return new AddrVal(1);
+			}
+		%}
+
+	function to_port(n: uint16, p: uint8): PortVal
+		%{
+		TransportProto proto = TRANSPORT_UNKNOWN;
+		switch ( p ) {
+		case 1: proto = TRANSPORT_ICMP; break;
+		case 6: proto = TRANSPORT_TCP; break;
+		case 17: proto = TRANSPORT_UDP; break;
+		}
+
+		return new PortVal(n, proto);
+		%}
+
+	#function proc_record(rec: Record) : bool
+	#	%{
+	#	return true;
+	#	%}
+
+	function proc_ids_event(ev: IDS_Event) : bool
+		%{
+		if ( ::unified2_event )
+			{
+			RecordVal* ids_event = new RecordVal(BifType::Record::Unified2::IDSEvent);
+			ids_event->Assign(0, new Val(${ev.sensor_id}, TYPE_COUNT));
+			ids_event->Assign(1, new Val(${ev.event_id}, TYPE_COUNT));
+			ids_event->Assign(2, new Val(ts_to_double(${ev.ts}), TYPE_TIME));
+			ids_event->Assign(3, new Val(${ev.signature_id}, TYPE_COUNT));
+			ids_event->Assign(4, new Val(${ev.generator_id}, TYPE_COUNT));
+			ids_event->Assign(5, new Val(${ev.signature_revision}, TYPE_COUNT));
+			ids_event->Assign(6, new Val(${ev.classification_id}, TYPE_COUNT));
+			ids_event->Assign(7, new Val(${ev.priority_id}, TYPE_COUNT));
+			ids_event->Assign(8, unified2_addr_to_bro_addr(${ev.src_ip}));
+			ids_event->Assign(9, unified2_addr_to_bro_addr(${ev.dst_ip}));
+			ids_event->Assign(10, to_port(${ev.src_p}, ${ev.protocol}));
+			ids_event->Assign(11, to_port(${ev.dst_p}, ${ev.protocol}));
+			ids_event->Assign(17, new Val(${ev.packet_action}, TYPE_COUNT));
+
+			val_list* vl = new val_list();
+			vl->append(connection()->bro_analyzer()->GetFile()->GetVal()->Ref());
+			vl->append(ids_event);
+			mgr.QueueEvent(::unified2_event, vl, SOURCE_LOCAL);
+			}
+		return true;
+		%}
+
+	function proc_ids_event_2(ev: IDS_Event_2) : bool
+		%{
+		if ( ::unified2_event )
+			{
+			RecordVal* ids_event = new RecordVal(BifType::Record::Unified2::IDSEvent);
+			ids_event->Assign(0, new Val(${ev.sensor_id}, TYPE_COUNT));
+			ids_event->Assign(1, new Val(${ev.event_id}, TYPE_COUNT));
+			ids_event->Assign(2, new Val(ts_to_double(${ev.ts}), TYPE_TIME));
+			ids_event->Assign(3, new Val(${ev.signature_id}, TYPE_COUNT));
+			ids_event->Assign(4, new Val(${ev.generator_id}, TYPE_COUNT));
+			ids_event->Assign(5, new Val(${ev.signature_revision}, TYPE_COUNT));
+			ids_event->Assign(6, new Val(${ev.classification_id}, TYPE_COUNT));
+			ids_event->Assign(7, new Val(${ev.priority_id}, TYPE_COUNT));
+			ids_event->Assign(8, unified2_addr_to_bro_addr(${ev.src_ip}));
+			ids_event->Assign(9, unified2_addr_to_bro_addr(${ev.dst_ip}));
+			ids_event->Assign(10, to_port(${ev.src_p}, ${ev.protocol}));
+			ids_event->Assign(11, to_port(${ev.dst_p}, ${ev.protocol}));
+			ids_event->Assign(12, new Val(${ev.impact_flag}, TYPE_COUNT));
+			ids_event->Assign(13, new Val(${ev.impact}, TYPE_COUNT));
+			ids_event->Assign(14, new Val(${ev.blocked}, TYPE_COUNT));
+			ids_event->Assign(15, new Val(${ev.mpls_label}, TYPE_COUNT));
+			ids_event->Assign(16, new Val(${ev.vlan_id}, TYPE_COUNT));
+
+			val_list* vl = new val_list();
+			vl->append(connection()->bro_analyzer()->GetFile()->GetVal()->Ref());
+			vl->append(ids_event);
+			mgr.QueueEvent(::unified2_event, vl, SOURCE_LOCAL);
+			}
+
+		return true;
+		%}
+
+	function proc_packet(pkt: Packet) : bool
+		%{
+		if ( ::unified2_packet )
+			{
+			RecordVal* packet = new RecordVal(BifType::Record::Unified2::Packet);
+			packet->Assign(0, new Val(${pkt.sensor_id}, TYPE_COUNT));
+			packet->Assign(1, new Val(${pkt.event_id}, TYPE_COUNT));
+			packet->Assign(2, new Val(${pkt.event_second}, TYPE_COUNT));
+			packet->Assign(3, new Val(ts_to_double(${pkt.packet_ts}), TYPE_TIME));
+			packet->Assign(4, new Val(${pkt.link_type}, TYPE_COUNT));
+			packet->Assign(5, bytestring_to_val(${pkt.packet_data}));
+
+			val_list* vl = new val_list();
+			vl->append(connection()->bro_analyzer()->GetFile()->GetVal()->Ref());
+			vl->append(packet);
+			mgr.QueueEvent(::unified2_packet, vl, SOURCE_LOCAL);
+			}
+
+		return true;
+		%}
+
+	#function proc_unknown_record_type(rec: UnknownRecordType) : bool
+	#	%{
+	#	printf("unknown packet type\n");
+	#	return true;
+	#	%}
+};
+
+#refine typeattr Record += &let {
+#	proc : bool = $context.flow.proc_record(this);
+#};
+
+refine typeattr IDS_Event += &let {
+	proc : bool = $context.flow.proc_ids_event(this);
+};
+
+refine typeattr IDS_Event_2 += &let {
+	proc : bool = $context.flow.proc_ids_event_2(this);
+};
+
+refine typeattr Packet += &let {
+	proc : bool = $context.flow.proc_packet(this);
+};
+
+#refine typeattr UnknownRecordType += &let {
+#	proc : bool = $context.flow.proc_unknown_record_type(this);
+#};

src/file_analysis/analyzer/unified2/unified2-file.pac

@@ -0,0 +1,91 @@
+
+enum Types {
+	PACKET           = 2,
+	IDS_EVENT        = 7,
+	IDS_EVENT_IPV6   = 72,
+	IDS_EVENT_2      = 104,
+	IDS_EVENT_IPV6_2 = 105,
+	EXTRA_DATA       = 110,
+};
+
+type Time = record {
+	seconds:      uint32;
+	microseconds: uint32;
+} &byteorder=bigendian;
+
+type Record = record {
+	rtype:   uint32;
+	length:  uint32;
+	data:   case rtype of {
+		PACKET            -> packet:              Packet(this);
+		IDS_EVENT         -> ids_event:           IDS_Event(this, 1);
+		IDS_EVENT_IPV6    -> ids_event_ipv6:      IDS_Event(this, 4);
+		IDS_EVENT_2       -> ids_event_vlan:      IDS_Event_2(this, 1);
+		IDS_EVENT_IPV6_2  -> ids_event_ipv6_vlan: IDS_Event_2(this, 4);
+		#EXTRA_DATA        -> extra_data:          ExtraData(this);
+		default           -> unknown_record_type: UnknownRecordType(this);
+	};
+} &byteorder=bigendian &length=length+8;
+
+type IDS_Event(rec: Record, ip_len: int) = record {
+	sensor_id:          uint32;
+	event_id:           uint32;
+	ts:                 Time;
+	signature_id:       uint32;
+	generator_id:       uint32;
+	signature_revision: uint32;
+	classification_id:  uint32;
+	priority_id:        uint32;
+	src_ip:             uint32[ip_len];
+	dst_ip:             uint32[ip_len];
+	src_p:              uint16;
+	dst_p:              uint16;
+	protocol:           uint8;
+	packet_action:      uint8;
+} &byteorder=bigendian;
+
+type IDS_Event_2(rec: Record, ip_len: int) = record {
+	sensor_id:          uint32;
+	event_id:           uint32;
+	ts:                 Time;
+	signature_id:       uint32;
+	generator_id:       uint32;
+	signature_revision: uint32;
+	classification_id:  uint32;
+	priority_id:        uint32;
+	src_ip:             uint32[ip_len];
+	dst_ip:             uint32[ip_len];
+	src_p:              uint16;
+	dst_p:              uint16;
+	protocol:           uint8;
+	impact_flag:        uint8;
+	impact:             uint8;
+	blocked:            uint8;
+	mpls_label:         uint32;
+	vlan_id:            uint16;
+	pad:                uint16;
+} &byteorder=bigendian;
+
+type Packet(rec: Record) = record {
+	sensor_id:          uint32;
+	event_id:           uint32;
+	event_second:       uint32;
+	packet_ts:          Time;
+	link_type:          uint32;
+	packet_len:         uint32;
+	packet_data:        bytestring &length=packet_len;
+} &byteorder=bigendian;
+
+type ExtraData(rec: Record) = record {
+	sensor_id:          uint32;
+	event_id:           uint32;
+	event_second:       uint32;
+	extra_type:         uint32;
+	data_type:          uint32;
+	blob_len:           uint32;
+	blob:               bytestring &length=blob_len;
+} &byteorder=bigendian &length=rec.length;
+
+type UnknownRecordType(rec: Record) = record {
+	data: bytestring &transient &length=rec.length;
+} &byteorder=bigendian &length=rec.length;

src/file_analysis/analyzer/unified2/unified2.pac

@@ -0,0 +1,21 @@
+
+%include binpac.pac
+%include bro.pac
+
+analyzer Unified2 withcontext {
+	analyzer:   Unified2_Analyzer;
+	flow:       Flow;
+};
+
+analyzer Unified2_Analyzer(bro_analyzer: BroFileAnalyzer) {
+	downflow = Flow;
+	upflow   = Flow;
+};
+
+%include unified2-file.pac
+
+flow Flow {
+	flowunit = Record withcontext(connection, this);
+};
+
+%include unified2-analyzer.pac