Merge branch 'master' into topic/jsiwek/misc

CHANGES

@@ -1,4 +1,15 @@
 
+2.1-1074 | 2013-08-14 10:29:54 -0700
+
+  * Fix timer type enum and timer name array mismatch. (Jon Siwek)
+
+2.1-1072 | 2013-08-14 10:28:51 -0700
+
+  * Adding the unified2 analyzer that reads unified2 files from disk,
+    turning them into events. (Seth Hall)
+
+  * Fixing intel framework tests. (Seth Hall)
+    
 2.1-1059 | 2013-08-13 23:52:41 -0400
 
   * Add file name support to intel framework. (Seth Hall)

VERSION

@@ -1 +1 @@
-2.1-1059
+2.1-1074

doc/scripts/DocSourcesList.cmake

@@ -71,6 +71,8 @@ rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_TCP.events.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_TCP.functions.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Teredo.events.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_UDP.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Unified2.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Unified2.types.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_ZIP.events.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/reporter.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/strings.bif.bro)
@@ -78,6 +80,7 @@ rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/top-k.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/types.bif.bro)
 rest_target(${psd} base/files/extract/main.bro)
 rest_target(${psd} base/files/hash/main.bro)
+rest_target(${psd} base/files/unified2/main.bro)
 rest_target(${psd} base/frameworks/analyzer/main.bro)
 rest_target(${psd} base/frameworks/cluster/main.bro)
 rest_target(${psd} base/frameworks/cluster/nodes/manager.bro)

scripts/base/files/unified2/__load__.bro

@@ -0,0 +1 @@
+@load ./main

scripts/base/files/unified2/main.bro

@@ -0,0 +1,244 @@
+
+@load base/utils/dir
+@load base/utils/paths
+
+module Unified2;
+
+export {
+	redef enum Log::ID += { LOG };
+
+	## Directory to watch for Unified2 files.
+	const watch_file = "" &redef;
+
+	## File to watch for Unified2 records.
+	const watch_dir = "" &redef;
+
+	## The sid-msg.map file you would like to use for your alerts.
+	const sid_msg = "" &redef;
+
+	## The gen-msg.map file you would like to use for your alerts.
+	const gen_msg = "" &redef;
+
+	## The classification.config file you would like to use for your alerts.
+	const classification_config = "" &redef;
+
+	## Reconstructed "alert" which combines related events
+	## and packets.
+	global alert: event(f: fa_file, ev: Unified2::IDSEvent, pkt: Unified2::Packet);
+
+	type PacketID: record {
+		src_ip: addr;
+		src_p: port;
+		dst_ip: addr;
+		dst_p: port;
+	} &log;
+
+	type Info: record {
+		## Timestamp attached to the alert.
+		ts:                 time     &log;
+		## Addresses and ports for the connection.
+		id:                 PacketID &log;
+		## Sensor that originated this event.
+		sensor_id:          count    &log;
+		## Sig id for this generator.
+		signature_id:       count    &log;
+		## A string representation of the "signature_id" field if a sid_msg.map file was loaded.
+		signature:          string   &log &optional;
+		## Which generator generated the alert?
+		generator_id:       count    &log;
+		## A string representation of the "generator_id" field if a gen_msg.map file was loaded.
+		generator:          string   &log &optional;
+		## Sig revision for this id.
+		signature_revision: count    &log;
+		## Event classification.
+		classification_id:  count    &log;
+		## Descriptive classification string,
+		classification:     string   &log &optional;
+		## Event priority.
+		priority_id:        count    &log;
+		## Event ID.
+		event_id:           count    &log;
+		## Some of the packet data.
+		packet:             string   &log &optional;
+	} &log;
+
+	## The event for accessing logged records.
+	global log_unified2: event(rec: Info);
+}
+
+# Mappings for extended information from alerts.
+global classification_map: table[count] of string;
+global sid_map: table[count] of string;
+global gen_map: table[count] of string;
+
+# For reading in config files.
+type OneLine: record {
+	line: string;
+};
+
+function create_info(ev: IDSEvent): Info
+	{
+	local info = Info($ts=ev$ts,
+	                  $id=PacketID($src_ip=ev$src_ip, $src_p=ev$src_p,
+	                               $dst_ip=ev$dst_ip, $dst_p=ev$dst_p),
+	                  $sensor_id=ev$sensor_id,
+	                  $signature_id=ev$signature_id,
+	                  $generator_id=ev$generator_id,
+	                  $signature_revision=ev$signature_revision,
+	                  $classification_id=ev$classification_id,
+	                  $priority_id=ev$priority_id,
+	                  $event_id=ev$event_id);
+
+	if ( ev$signature_id in sid_map )
+		info$signature=sid_map[ev$signature_id];
+	if ( ev$generator_id in gen_map )
+		info$generator=gen_map[ev$generator_id];
+	if ( ev$classification_id in classification_map )
+		info$classification=classification_map[ev$classification_id];
+
+	return info;
+	}
+
+redef record fa_file += {
+	## Recently received IDS events.  This is primarily used
+	## for tying together Unified2 events and packets.
+	u2_events: table[count] of Unified2::IDSEvent
+		&optional &create_expire=5sec
+		&expire_func=function(t: table[count] of Unified2::IDSEvent, event_id: count): interval
+			{
+			Log::write(LOG, create_info(t[event_id]));
+			return 0secs;
+			};
+};
+
+event Unified2::read_sid_msg_line(desc: Input::EventDescription, tpe: Input::Event, line: string)
+	{
+	local parts = split_n(line, / \|\| /, F, 100);
+	if ( |parts| >= 2 && /^[0-9]+$/ in parts[1] )
+		sid_map[to_count(parts[1])] = parts[2];
+	}
+
+event Unified2::read_gen_msg_line(desc: Input::EventDescription, tpe: Input::Event, line: string)
+	{
+	local parts = split_n(line, / \|\| /, F, 3);
+	if ( |parts| >= 2 && /^[0-9]+$/ in parts[1] )
+		gen_map[to_count(parts[1])] = parts[3];
+	}
+
+event Unified2::read_classification_line(desc: Input::EventDescription, tpe: Input::Event, line: string)
+	{
+	local parts = split_n(line, /: /, F, 2);
+	if ( |parts| == 2 )
+		{
+		local parts2 = split_n(parts[2], /,/, F, 4);
+		if ( |parts2| > 1 )
+			classification_map[|classification_map|+1] = parts2[1];
+		}
+	}
+
+event bro_init() &priority=5
+	{
+	Log::create_stream(Unified2::LOG, [$columns=Info, $ev=log_unified2]);
+
+	if ( sid_msg != "" )
+		{
+		Input::add_event([$source=sid_msg,
+		                  $reader=Input::READER_RAW,
+		                  $mode=Input::REREAD,
+		                  $name=sid_msg,
+		                  $fields=Unified2::OneLine,
+		                  $want_record=F,
+		                  $ev=Unified2::read_sid_msg_line]);
+		}
+
+	if ( gen_msg != "" )
+		{
+		Input::add_event([$source=gen_msg,
+		                  $name=gen_msg,
+		                  $reader=Input::READER_RAW,
+		                  $mode=Input::REREAD,
+		                  $fields=Unified2::OneLine,
+		                  $want_record=F,
+		                  $ev=Unified2::read_gen_msg_line]);
+		}
+
+	if ( classification_config != "" )
+		{
+		Input::add_event([$source=classification_config,
+		                  $name=classification_config,
+		                  $reader=Input::READER_RAW,
+		                  $mode=Input::REREAD,
+		                  $fields=Unified2::OneLine,
+		                  $want_record=F,
+		                  $ev=Unified2::read_classification_line]);
+		}
+
+	if ( watch_dir != "" )
+		{
+		Dir::monitor(watch_dir, function(fname: string)
+			{
+			Input::add_analysis([$source=fname,
+			                     $reader=Input::READER_BINARY,
+			                     $mode=Input::STREAM,
+			                     $name=fname]);
+			}, 10secs);
+		}
+
+	if ( watch_file != "" )
+		{
+		Input::add_analysis([$source=watch_file,
+		                     $reader=Input::READER_BINARY,
+		                     $mode=Input::STREAM,
+		                     $name=watch_file]);
+		}
+	}
+
+event file_new(f: fa_file)
+	{
+	local file_dir = "";
+	local parts = split_all(f$source, /\/[^\/]*$/);
+	if ( |parts| == 3 )
+		file_dir = parts[1];
+
+	if ( (watch_file != "" && f$source == watch_file) || 
+	     (watch_dir != "" && compress_path(watch_dir) == file_dir) )
+		{
+		Files::add_analyzer(f, Files::ANALYZER_UNIFIED2);
+		f$u2_events = table();
+		}
+	}
+
+event unified2_event(f: fa_file, ev: Unified2::IDSEvent)
+	{
+	f$u2_events[ev$event_id] = ev;
+	}
+
+event unified2_packet(f: fa_file, pkt: Unified2::Packet)
+	{
+	if ( f?$u2_events && pkt$event_id in f$u2_events)
+		{
+		local ev = f$u2_events[pkt$event_id];
+		event Unified2::alert(f, ev, pkt);
+		delete f$u2_events[pkt$event_id];
+		}
+	}
+
+event Unified2::alert(f: fa_file, ev: IDSEvent, pkt: Packet)
+	{
+	local info = create_info(ev);
+	info$packet=pkt$data;
+	Log::write(LOG, info);
+	}
+
+event file_state_remove(f: fa_file)
+	{
+	if ( f?$u2_events )
+		{
+		# In case any events never had matching packets, flush
+		# the extras to the log.
+		for ( i in f$u2_events )
+			{
+			Log::write(LOG, create_info(f$u2_events[i]));
+			}
+		}
+	}

scripts/base/init-bare.bro

@@ -2686,6 +2686,42 @@ type ModbusHeaders: record {
 	function_code: count;
 };
 
+module Unified2;
+export {
+	type Unified2::IDSEvent: record {
+		sensor_id:          count;
+		event_id:           count;
+		ts:                 time;
+		signature_id:       count;
+		generator_id:       count;
+		signature_revision: count;
+		classification_id:  count;
+		priority_id:        count;
+		src_ip:             addr;
+		dst_ip:             addr;
+		src_p:              port;
+		dst_p:              port;
+		impact_flag:        count;
+		impact:             count;
+		blocked:            count;
+		## Not available in "legacy" IDS events.
+		mpls_label:         count  &optional;
+		## Not available in "legacy" IDS events.
+		vlan_id:            count  &optional;
+		## Only available in "legacy" IDS events.
+		packet_action:      count  &optional;
+	};
+
+	type Unified2::Packet: record {
+		sensor_id:    count;
+		event_id:     count;
+		event_second: count;
+		packet_ts:    time;
+		link_type:    count;
+		data:         string;
+	};
+}
+
 module SOCKS;
 export {
 	## This record is for a SOCKS client or server to provide either a

scripts/base/init-default.bro

@@ -56,5 +56,7 @@
 
 @load base/files/hash
 @load base/files/extract
+@load base/files/unified2
+
 
 @load base/misc/find-checksum-offloading

src/Timer.cc

@@ -16,6 +16,7 @@ const char* TimerNames[] = {
 	"ConnectionInactivityTimer",
 	"ConnectionStatusUpdateTimer",
 	"DNSExpireTimer",
+	"FileAnalysisInactivityTimer",
 	"FragTimer",
 	"IncrementalSendTimer",
 	"IncrementalWriteTimer",

src/Timer.h

@@ -23,7 +23,6 @@ enum TimerType {
 	TIMER_CONN_STATUS_UPDATE,
 	TIMER_DNS_EXPIRE,
 	TIMER_FILE_ANALYSIS_INACTIVITY,
-	TIMER_FILE_ANALYSIS_DRAIN,
 	TIMER_FRAG,
 	TIMER_INCREMENTAL_SEND,
 	TIMER_INCREMENTAL_WRITE,

src/binpac_bro.h

@@ -12,6 +12,7 @@ namespace analyzer { class Analyzer; }
 #include "event.bif.func_h"
 #include "TunnelEncapsulation.h"
 #include "analyzer/Analyzer.h"
+#include "file_analysis/Analyzer.h"
 #include "Conn.h"
 
 #include "binpac.h"
@@ -19,6 +20,7 @@ namespace analyzer { class Analyzer; }
 namespace binpac {
 
 typedef analyzer::Analyzer* BroAnalyzer;
+typedef file_analysis::Analyzer BroFileAnalyzer;
 typedef Val* BroVal;
 typedef PortVal* BroPortVal;
 typedef StringVal* BroStringVal;

src/file_analysis/analyzer/CMakeLists.txt

@@ -1,3 +1,4 @@
 add_subdirectory(data_event)
 add_subdirectory(extract)
 add_subdirectory(hash)
+add_subdirectory(unified2)

src/file_analysis/analyzer/unified2/CMakeLists.txt

@@ -0,0 +1,11 @@
+
+include(BroPlugin)
+
+include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}
+                           ${CMAKE_CURRENT_BINARY_DIR})
+
+bro_plugin_begin(Bro Unified2)
+bro_plugin_cc(Unified2.cc Plugin.cc ../../Analyzer.cc)
+bro_plugin_bif(events.bif types.bif)
+bro_plugin_pac(unified2.pac unified2-file.pac unified2-analyzer.pac)
+bro_plugin_end()

src/file_analysis/analyzer/unified2/Plugin.cc

@@ -0,0 +1,35 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "plugin/Plugin.h"
+
+#include "file_analysis/Component.h"
+
+#include "Unified2.h"
+
+namespace plugin { namespace Bro_Unified2 {
+
+class Plugin : public plugin::Plugin {
+protected:
+	void InitPreScript()
+		{
+		SetName("Bro::Unified2");
+		SetVersion(-1);
+		SetAPIVersion(BRO_PLUGIN_API_VERSION);
+		SetDynamicPlugin(false);
+
+		SetDescription("Analyze Unified2 alert files.");
+
+		AddComponent(new ::file_analysis::Component("UNIFIED2",
+		        ::file_analysis::Unified2::Instantiate));
+
+		extern std::list<std::pair<const char*, int> > __bif_events_init();
+		AddBifInitFunction(&__bif_events_init);
+
+		extern std::list<std::pair<const char*, int> > __bif_types_init();
+		AddBifInitFunction(&__bif_types_init);
+		}
+};
+
+Plugin __plugin;
+
+} }

src/file_analysis/analyzer/unified2/Unified2.cc

@@ -0,0 +1,38 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "Unified2.h"
+#include "file_analysis/Manager.h"
+
+using namespace file_analysis;
+
+Unified2::Unified2(RecordVal* args, File* file)
+    : file_analysis::Analyzer(file_mgr->GetComponentTag("UNIFIED2"), args, file)
+	{
+	interp = new binpac::Unified2::Unified2_Analyzer(this);
+	}
+
+Unified2::~Unified2()
+	{
+	delete interp;
+	}
+
+file_analysis::Analyzer* Unified2::Instantiate(RecordVal* args, File* file)
+	{
+	return new Unified2(args, file);
+	}
+
+bool Unified2::DeliverStream(const u_char* data, uint64 len)
+	{
+	try
+		{
+		interp->NewData(true, data, data + len);
+		}
+
+	catch ( const binpac::Exception& e )
+		{
+		printf("Binpac exception: %s\n", e.c_msg());
+		return false;
+		}
+
+	return true;
+	}

src/file_analysis/analyzer/unified2/Unified2.h

@@ -0,0 +1,38 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#ifndef FILE_ANALYSIS_UNIFIED2_H
+#define FILE_ANALYSIS_UNIFIED2_H
+
+#include <string>
+
+#include "Val.h"
+#include "File.h"
+#include "Analyzer.h"
+#include "unified2_pac.h"
+
+namespace file_analysis {
+
+/**
+ * An analyzer to extract content of files from local disk.
+ */
+class Unified2 : public file_analysis::Analyzer {
+public:
+	virtual ~Unified2();
+
+	virtual bool DeliverStream(const u_char* data, uint64 len);
+
+	static file_analysis::Analyzer* Instantiate(RecordVal* args, File* file);
+
+protected:
+	Unified2(RecordVal* args, File* file);
+
+private:
+	binpac::Unified2::Unified2_Analyzer* interp;
+
+	string filename;
+	int fd;
+};
+
+} // namespace file_analysis
+
+#endif

src/file_analysis/analyzer/unified2/events.bif

@@ -0,0 +1,7 @@
+
+## Abstract all of the various Unified2 event formats into 
+## a single event.
+event unified2_event%(f: fa_file, ev: Unified2::IDSEvent%);
+
+## The Unified2 packet format event.
+event unified2_packet%(f: fa_file, pkt: Unified2::Packet%);

src/file_analysis/analyzer/unified2/types.bif

@@ -0,0 +1,2 @@
+type Unified2::IDSEvent: record;
+type Unified2::Packet: record;

src/file_analysis/analyzer/unified2/unified2-analyzer.pac

@@ -0,0 +1,170 @@
+
+%extern{
+#include "Event.h"
+#include "file_analysis/File.h"
+#include "events.bif.h"
+#include "types.bif.h"
+#include "IPAddr.h"
+%}
+
+refine flow Flow += {
+
+	%member{
+	%}
+
+	%init{
+	%}
+
+	%eof{
+	%}
+
+	%cleanup{
+	%}
+
+	function ts_to_double(ts: Time): double
+		%{
+		double t = ${ts.seconds} + (${ts.microseconds} / 1000000);
+		return t;
+		%}
+
+	function unified2_addr_to_bro_addr(a: uint32[]): AddrVal
+		%{
+		if ( a->size() == 1 )
+			{
+			return new AddrVal(IPAddr(IPv4, &(a->at(0)), IPAddr::Host));
+			}
+		else if ( a->size() == 4 )
+			{
+			uint32 tmp[4] = { a->at(0), a->at(1), a->at(2), a->at(3) };
+			return new AddrVal(IPAddr(IPv6, tmp, IPAddr::Host));
+			}
+		else
+			{
+			// Should never reach here.
+			return new AddrVal(1);
+			}
+		%}
+
+	function to_port(n: uint16, p: uint8): PortVal
+		%{
+		TransportProto proto = TRANSPORT_UNKNOWN;
+		switch ( p ) {
+		case 1: proto = TRANSPORT_ICMP; break;
+		case 6: proto = TRANSPORT_TCP; break;
+		case 17: proto = TRANSPORT_UDP; break;
+		}
+
+		return new PortVal(n, proto);
+		%}
+
+	#function proc_record(rec: Record) : bool
+	#	%{
+	#	return true;
+	#	%}
+
+	function proc_ids_event(ev: IDS_Event) : bool
+		%{
+		if ( ::unified2_event )
+			{
+			RecordVal* ids_event = new RecordVal(BifType::Record::Unified2::IDSEvent);
+			ids_event->Assign(0, new Val(${ev.sensor_id}, TYPE_COUNT));
+			ids_event->Assign(1, new Val(${ev.event_id}, TYPE_COUNT));
+			ids_event->Assign(2, new Val(ts_to_double(${ev.ts}), TYPE_TIME));
+			ids_event->Assign(3, new Val(${ev.signature_id}, TYPE_COUNT));
+			ids_event->Assign(4, new Val(${ev.generator_id}, TYPE_COUNT));
+			ids_event->Assign(5, new Val(${ev.signature_revision}, TYPE_COUNT));
+			ids_event->Assign(6, new Val(${ev.classification_id}, TYPE_COUNT));
+			ids_event->Assign(7, new Val(${ev.priority_id}, TYPE_COUNT));
+			ids_event->Assign(8, unified2_addr_to_bro_addr(${ev.src_ip}));
+			ids_event->Assign(9, unified2_addr_to_bro_addr(${ev.dst_ip}));
+			ids_event->Assign(10, to_port(${ev.src_p}, ${ev.protocol}));
+			ids_event->Assign(11, to_port(${ev.dst_p}, ${ev.protocol}));
+			ids_event->Assign(17, new Val(${ev.packet_action}, TYPE_COUNT));
+
+			val_list* vl = new val_list();
+			vl->append(connection()->bro_analyzer()->GetFile()->GetVal()->Ref());
+			vl->append(ids_event);
+			mgr.QueueEvent(::unified2_event, vl, SOURCE_LOCAL);
+			}
+		return true;
+		%}
+
+	function proc_ids_event_2(ev: IDS_Event_2) : bool
+		%{
+		if ( ::unified2_event )
+			{
+			RecordVal* ids_event = new RecordVal(BifType::Record::Unified2::IDSEvent);
+			ids_event->Assign(0, new Val(${ev.sensor_id}, TYPE_COUNT));
+			ids_event->Assign(1, new Val(${ev.event_id}, TYPE_COUNT));
+			ids_event->Assign(2, new Val(ts_to_double(${ev.ts}), TYPE_TIME));
+			ids_event->Assign(3, new Val(${ev.signature_id}, TYPE_COUNT));
+			ids_event->Assign(4, new Val(${ev.generator_id}, TYPE_COUNT));
+			ids_event->Assign(5, new Val(${ev.signature_revision}, TYPE_COUNT));
+			ids_event->Assign(6, new Val(${ev.classification_id}, TYPE_COUNT));
+			ids_event->Assign(7, new Val(${ev.priority_id}, TYPE_COUNT));
+			ids_event->Assign(8, unified2_addr_to_bro_addr(${ev.src_ip}));
+			ids_event->Assign(9, unified2_addr_to_bro_addr(${ev.dst_ip}));
+			ids_event->Assign(10, to_port(${ev.src_p}, ${ev.protocol}));
+			ids_event->Assign(11, to_port(${ev.dst_p}, ${ev.protocol}));
+			ids_event->Assign(12, new Val(${ev.impact_flag}, TYPE_COUNT));
+			ids_event->Assign(13, new Val(${ev.impact}, TYPE_COUNT));
+			ids_event->Assign(14, new Val(${ev.blocked}, TYPE_COUNT));
+			ids_event->Assign(15, new Val(${ev.mpls_label}, TYPE_COUNT));
+			ids_event->Assign(16, new Val(${ev.vlan_id}, TYPE_COUNT));
+
+			val_list* vl = new val_list();
+			vl->append(connection()->bro_analyzer()->GetFile()->GetVal()->Ref());
+			vl->append(ids_event);
+			mgr.QueueEvent(::unified2_event, vl, SOURCE_LOCAL);
+			}
+
+		return true;
+		%}
+
+	function proc_packet(pkt: Packet) : bool
+		%{
+		if ( ::unified2_packet )
+			{
+			RecordVal* packet = new RecordVal(BifType::Record::Unified2::Packet);
+			packet->Assign(0, new Val(${pkt.sensor_id}, TYPE_COUNT));
+			packet->Assign(1, new Val(${pkt.event_id}, TYPE_COUNT));
+			packet->Assign(2, new Val(${pkt.event_second}, TYPE_COUNT));
+			packet->Assign(3, new Val(ts_to_double(${pkt.packet_ts}), TYPE_TIME));
+			packet->Assign(4, new Val(${pkt.link_type}, TYPE_COUNT));
+			packet->Assign(5, bytestring_to_val(${pkt.packet_data}));
+
+			val_list* vl = new val_list();
+			vl->append(connection()->bro_analyzer()->GetFile()->GetVal()->Ref());
+			vl->append(packet);
+			mgr.QueueEvent(::unified2_packet, vl, SOURCE_LOCAL);
+			}
+
+		return true;
+		%}
+
+	#function proc_unknown_record_type(rec: UnknownRecordType) : bool
+	#	%{
+	#	printf("unknown packet type\n");
+	#	return true;
+	#	%}
+};
+
+#refine typeattr Record += &let {
+#	proc : bool = $context.flow.proc_record(this);
+#};
+
+refine typeattr IDS_Event += &let {
+	proc : bool = $context.flow.proc_ids_event(this);
+};
+
+refine typeattr IDS_Event_2 += &let {
+	proc : bool = $context.flow.proc_ids_event_2(this);
+};
+
+refine typeattr Packet += &let {
+	proc : bool = $context.flow.proc_packet(this);
+};
+
+#refine typeattr UnknownRecordType += &let {
+#	proc : bool = $context.flow.proc_unknown_record_type(this);
+#};

src/file_analysis/analyzer/unified2/unified2-file.pac

@@ -0,0 +1,91 @@
+
+enum Types {
+	PACKET           = 2,
+	IDS_EVENT        = 7,
+	IDS_EVENT_IPV6   = 72,
+	IDS_EVENT_2      = 104,
+	IDS_EVENT_IPV6_2 = 105,
+	EXTRA_DATA       = 110,
+};
+
+type Time = record {
+	seconds:      uint32;
+	microseconds: uint32;
+} &byteorder=bigendian;
+
+type Record = record {
+	rtype:   uint32;
+	length:  uint32;
+	data:   case rtype of {
+		PACKET            -> packet:              Packet(this);
+		IDS_EVENT         -> ids_event:           IDS_Event(this, 1);
+		IDS_EVENT_IPV6    -> ids_event_ipv6:      IDS_Event(this, 4);
+		IDS_EVENT_2       -> ids_event_vlan:      IDS_Event_2(this, 1);
+		IDS_EVENT_IPV6_2  -> ids_event_ipv6_vlan: IDS_Event_2(this, 4);
+		#EXTRA_DATA        -> extra_data:          ExtraData(this);
+		default           -> unknown_record_type: UnknownRecordType(this);
+	};
+} &byteorder=bigendian &length=length+8;
+
+type IDS_Event(rec: Record, ip_len: int) = record {
+	sensor_id:          uint32;
+	event_id:           uint32;
+	ts:                 Time;
+	signature_id:       uint32;
+	generator_id:       uint32;
+	signature_revision: uint32;
+	classification_id:  uint32;
+	priority_id:        uint32;
+	src_ip:             uint32[ip_len];
+	dst_ip:             uint32[ip_len];
+	src_p:              uint16;
+	dst_p:              uint16;
+	protocol:           uint8;
+	packet_action:      uint8;
+} &byteorder=bigendian;
+
+type IDS_Event_2(rec: Record, ip_len: int) = record {
+	sensor_id:          uint32;
+	event_id:           uint32;
+	ts:                 Time;
+	signature_id:       uint32;
+	generator_id:       uint32;
+	signature_revision: uint32;
+	classification_id:  uint32;
+	priority_id:        uint32;
+	src_ip:             uint32[ip_len];
+	dst_ip:             uint32[ip_len];
+	src_p:              uint16;
+	dst_p:              uint16;
+	protocol:           uint8;
+	impact_flag:        uint8;
+	impact:             uint8;
+	blocked:            uint8;
+	mpls_label:         uint32;
+	vlan_id:            uint16;
+	pad:                uint16;
+} &byteorder=bigendian;
+
+type Packet(rec: Record) = record {
+	sensor_id:          uint32;
+	event_id:           uint32;
+	event_second:       uint32;
+	packet_ts:          Time;
+	link_type:          uint32;
+	packet_len:         uint32;
+	packet_data:        bytestring &length=packet_len;
+} &byteorder=bigendian;
+
+type ExtraData(rec: Record) = record {
+	sensor_id:          uint32;
+	event_id:           uint32;
+	event_second:       uint32;
+	extra_type:         uint32;
+	data_type:          uint32;
+	blob_len:           uint32;
+	blob:               bytestring &length=blob_len;
+} &byteorder=bigendian &length=rec.length;
+
+type UnknownRecordType(rec: Record) = record {
+	data: bytestring &transient &length=rec.length;
+} &byteorder=bigendian &length=rec.length;

src/file_analysis/analyzer/unified2/unified2.pac

@@ -0,0 +1,21 @@
+
+%include binpac.pac
+%include bro.pac
+
+analyzer Unified2 withcontext {
+	analyzer:   Unified2_Analyzer;
+	flow:       Flow;
+};
+
+analyzer Unified2_Analyzer(bro_analyzer: BroFileAnalyzer) {
+	downflow = Flow;
+	upflow   = Flow;
+};
+
+%include unified2-file.pac
+
+flow Flow {
+	flowunit = Record withcontext(connection, this);
+};
+
+%include unified2-analyzer.pac

testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log

@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
-#open	2013-08-12-18-22-48
+#open	2013-08-14-01-19-27
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@@ -62,6 +62,8 @@ scripts/base/init-bare.bro
     build/scripts/base/bif/plugins/Bro_TCP.functions.bif.bro
     build/scripts/base/bif/plugins/Bro_Teredo.events.bif.bro
     build/scripts/base/bif/plugins/Bro_UDP.events.bif.bro
+    build/scripts/base/bif/plugins/Bro_Unified2.events.bif.bro
+    build/scripts/base/bif/plugins/Bro_Unified2.types.bif.bro
     build/scripts/base/bif/plugins/Bro_ZIP.events.bif.bro
   scripts/base/frameworks/logging/__load__.bro
     scripts/base/frameworks/logging/main.bro
@@ -95,4 +97,4 @@ scripts/base/init-bare.bro
     build/scripts/base/bif/top-k.bif.bro
 scripts/policy/misc/loaded-scripts.bro
   scripts/base/utils/paths.bro
-#close	2013-08-12-18-22-48
+#close	2013-08-14-01-19-27

testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log

@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
-#open	2013-08-12-18-22-49
+#open	2013-08-14-01-19-28
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@@ -62,6 +62,8 @@ scripts/base/init-bare.bro
     build/scripts/base/bif/plugins/Bro_TCP.functions.bif.bro
     build/scripts/base/bif/plugins/Bro_Teredo.events.bif.bro
     build/scripts/base/bif/plugins/Bro_UDP.events.bif.bro
+    build/scripts/base/bif/plugins/Bro_Unified2.events.bif.bro
+    build/scripts/base/bif/plugins/Bro_Unified2.types.bif.bro
     build/scripts/base/bif/plugins/Bro_ZIP.events.bif.bro
   scripts/base/frameworks/logging/__load__.bro
     scripts/base/frameworks/logging/main.bro
@@ -210,6 +212,8 @@ scripts/base/init-default.bro
     scripts/base/files/hash/main.bro
   scripts/base/files/extract/__load__.bro
     scripts/base/files/extract/main.bro
+  scripts/base/files/unified2/__load__.bro
+    scripts/base/files/unified2/main.bro
   scripts/base/misc/find-checksum-offloading.bro
 scripts/policy/misc/loaded-scripts.bro
-#close	2013-08-12-18-22-49
+#close	2013-08-14-01-19-28

testing/btest/Baseline/scripts.base.files.unified2.alert/unified2.log

@@ -0,0 +1,11 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	unified2
+#open	2013-08-13-07-16-01
+#fields	ts	id.src_ip	id.src_p	id.dst_ip	id.dst_p	sensor_id	signature_id	signature	generator_id	generator	signature_revision	classification_id	classification	priority_id	event_id	packet
+#types	time	addr	port	addr	port	count	count	string	count	string	count	count	string	count	count	string
+1323827323.000000	192.168.1.72	50185	74.125.225.49	80	0	2003058	ET MALWARE 180solutions (Zango) Spyware Installer Download	1	snort general alert	5	21	trojan-activity	1	2	\xd80bH\xc5\xb5x\xca9\xb7\xe4r\x08\x00E\x10\x00\\x1a\xce@\x00@\x062\x1f\xc0\xa8\x01HJ}\xe11\xc4\x09\x00P*\xa8bv]z/\xde\x80\x18\x82+\x88,\x00\x00\x01\x01\x08\x0a\x17J\x83Q\xfe\xad\xac\x1aGET /Zango/ZangoInstaller.exe HTTP/1.0\x0d\x0a
+1323827344.000000	192.168.1.72	49862	199.47.216.144	80	0	2012647	ET POLICY Dropbox.com Offsite File Backup in Use	1	snort general alert	3	33	policy-violation	1	3	\xd80bH\xc5\xb5x\xca9\xb7\xe4r\x08\x00E\x00\x00\xf8Q\xdf@\x00@\x06\x86p\xc0\xa8\x01H\xc7/\xd8\x90\xc2\xc6\x00P\x9cm\x97U\xf07\x084\x80\x18\x82\x18%<\x00\x00\x01\x01\x08\x0a\x17J\xd7\xde\x00\x92\x81\xc5GET /subscribe?host_int=43112345&ns_map=123456_1234524412104916591&ts=1323827344 HTTP/1.1\x0d\x0aHost: notify1.dropbox.com\x0d\x0aAccept-Encoding: identity\x0d\x0aConnection: keep-alive\x0d\x0aX-Dropbox-Locale: en_US\x0d\x0a\x0d\x0a
+#close	2013-08-13-07-16-01

testing/btest/Baseline/scripts.base.frameworks.file-analysis.logging/files.log

@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	files
-#open	2013-07-25-16-57-31
+#open	2013-08-14-04-50-17
 #fields	ts	fuid	tx_hosts	rx_hosts	conn_uids	source	depth	analyzers	mime_type	filename	duration	local_orig	is_orig	seen_bytes	total_bytes	missing_bytes	overflow_bytes	timedout	parent_fuid	md5	sha1	sha256	extracted
 #types	time	string	table[addr]	table[addr]	table[string]	string	count	table[string]	string	string	interval	bool	bool	count	count	count	count	bool	string	string	string	string	string
 1362692527.009721	G75mcAsU764	192.150.187.43	141.142.228.5	UWkUyAuUGXf	HTTP	0	SHA256,DATA_EVENT,MD5,EXTRACT,SHA1	text/plain	-	0.000054	-	F	4705	4705	0	0	F	-	397168fd09991a0e712254df7bc639ac	1dd7ac0398df6cbc0696445a91ec681facf4dc47	4e7c7ef0984119447e743e3ec77e1de52713e345cde03fe7df753a35849bed18	G75mcAsU764-file
-#close	2013-07-25-16-57-31
+#close	2013-08-14-04-50-17

testing/btest/btest.cfg

@@ -14,6 +14,7 @@ LC_ALL=C
 BTEST_PATH=%(testbase)s/../../aux/btest
 PATH=%(testbase)s/../../build/src:%(testbase)s/../scripts:%(testbase)s/../../aux/btest:%(testbase)s/../../aux/bro-aux/bro-cut:%(default_path)s
 TRACES=%(testbase)s/Traces
+FILES=%(testbase)s/Files
 SCRIPTS=%(testbase)s/../scripts
 DIST=%(testbase)s/../..
 BUILD=%(testbase)s/../../build

testing/btest/scripts/base/files/unified2/alert.bro

@@ -0,0 +1,76 @@
+# @TEST-EXEC: bro -b %INPUT Unified2::watch_file=$FILES/unified2.u2
+# @TEST-EXEC: btest-diff unified2.log
+
+@TEST-START-FILE sid_msg.map
+2003058 || ET MALWARE 180solutions (Zango) Spyware Installer Download || url,doc.emergingthreats.net/bin/view/Main/2003058 || url,securityresponse.symantec.com/avcenter/venc/data/pf/adware.180search.html
+2012647 || ET POLICY Dropbox.com Offsite File Backup in Use || url,dereknewton.com/2011/04/dropbox-authentication-static-host-ids/ || url,www.dropbox.com
+@TEST-END-FILE
+
+@TEST-START-FILE gen_msg.map
+1 || 1 || snort general alert
+2 || 1 || tag: Tagged Packet
+3 || 1 || snort dynamic alert
+100 || 1 || spp_portscan: Portscan Detected
+100 || 2 || spp_portscan: Portscan Status
+100 || 3 || spp_portscan: Portscan Ended
+101 || 1 || spp_minfrag: minfrag alert
+@TEST-END-FILE
+
+@TEST-START-FILE classification.config
+#
+# config classification:shortname,short description,priority
+#
+
+#Traditional classifications. These will be replaced soon
+
+config classification: not-suspicious,Not Suspicious Traffic,3
+config classification: unknown,Unknown Traffic,3
+config classification: bad-unknown,Potentially Bad Traffic, 2
+config classification: attempted-recon,Attempted Information Leak,2
+config classification: successful-recon-limited,Information Leak,2
+config classification: successful-recon-largescale,Large Scale Information Leak,2
+config classification: attempted-dos,Attempted Denial of Service,2
+config classification: successful-dos,Denial of Service,2
+config classification: attempted-user,Attempted User Privilege Gain,1
+config classification: unsuccessful-user,Unsuccessful User Privilege Gain,1
+config classification: successful-user,Successful User Privilege Gain,1
+config classification: attempted-admin,Attempted Administrator Privilege Gain,1
+config classification: successful-admin,Successful Administrator Privilege Gain,1
+config classification: rpc-portmap-decode,Decode of an RPC Query,2
+config classification: shellcode-detect,Executable Code was Detected,1
+config classification: string-detect,A Suspicious String was Detected,3
+config classification: suspicious-filename-detect,A Suspicious Filename was Detected,2
+config classification: suspicious-login,An Attempted Login Using a Suspicious Username was Detected,2
+config classification: system-call-detect,A System Call was Detected,2
+config classification: tcp-connection,A TCP Connection was Detected,4
+config classification: trojan-activity,A Network Trojan was Detected, 1
+config classification: unusual-client-port-connection,A Client was Using an Unusual Port,2
+config classification: network-scan,Detection of a Network Scan,3
+config classification: denial-of-service,Detection of a Denial of Service Attack,2
+config classification: non-standard-protocol,Detection of a Non-Standard Protocol or Event,2
+config classification: protocol-command-decode,Generic Protocol Command Decode,3
+config classification: web-application-activity,Access to a Potentially Vulnerable Web Application,2
+config classification: web-application-attack,Web Application Attack,1
+config classification: misc-activity,Misc activity,3
+config classification: misc-attack,Misc Attack,2
+config classification: icmp-event,Generic ICMP event,3
+config classification: inappropriate-content,Inappropriate Content was Detected,1
+config classification: policy-violation,Potential Corporate Privacy Violation,1
+config classification: default-login-attempt,Attempt to Login By a Default Username and Password,2
+@TEST-END-FILE
+
+redef exit_only_after_terminate = T;
+
+@load base/files/unified2
+
+redef Unified2::sid_msg = @DIR+"/sid_msg.map";
+redef Unified2::gen_msg = @DIR+"/gen_msg.map";
+redef Unified2::classification_config = @DIR+"/classification.config";
+global i = 0;
+
+event Unified2::alert(f: fa_file, ev: Unified2::IDSEvent, pkt: Unified2::Packet)
+	{
+	++i;
+	if ( i == 2 )
+		terminate();
+	}