Merge remote-tracking branch 'origin/master' into topic/seth/smb

# Conflicts:
#	scripts/base/init-default.bro

scripts/base/files/extract/main.bro

@@ -53,7 +53,8 @@ function set_limit(f: fa_file, args: Files::AnalyzerArgs, n: count): bool
 function on_add(f: fa_file, args: Files::AnalyzerArgs)
 	{
 	if ( ! args?$extract_filename )
-		args$extract_filename = cat("extract-", f$source, "-", f$id);
+		args$extract_filename = cat("extract-", f$last_active, "-", f$source,
+		                            "-", f$id);
 
 	f$info$extracted = args$extract_filename;
 	args$extract_filename = build_path_compressed(prefix, args$extract_filename);

scripts/base/files/pe/README

@@ -0,0 +1 @@
+Support for Portable Executable (PE) file analysis.

scripts/base/files/pe/__load__.bro

@@ -0,0 +1,2 @@
+@load ./consts
+@load ./main

scripts/base/files/pe/consts.bro

@@ -0,0 +1,184 @@
+
+module PE;
+
+export {
+	const machine_types: table[count] of string = {
+		[0x00]   = "UNKNOWN",
+		[0x1d3]  = "AM33",
+		[0x8664] = "AMD64",
+		[0x1c0]  = "ARM",
+		[0x1c4]  = "ARMNT",
+		[0xaa64] = "ARM64",
+		[0xebc]  = "EBC",
+		[0x14c]  = "I386",
+		[0x200]  = "IA64",
+		[0x9041] = "M32R",
+		[0x266]  = "MIPS16",
+		[0x366]  = "MIPSFPU",
+		[0x466]  = "MIPSFPU16",
+		[0x1f0]  = "POWERPC",
+		[0x1f1]  = "POWERPCFP",
+		[0x166]  = "R4000",
+		[0x1a2]  = "SH3",
+		[0x1a3]  = "SH3DSP",
+		[0x1a6]  = "SH4",
+		[0x1a8]  = "SH5",
+		[0x1c2]  = "THUMB",
+		[0x169]  = "WCEMIPSV2"
+	} &default=function(i: count):string { return fmt("unknown-%d", i); };
+
+	const file_characteristics: table[count] of string = {
+		[0x1]    = "RELOCS_STRIPPED",
+		[0x2]    = "EXECUTABLE_IMAGE",
+		[0x4]    = "LINE_NUMS_STRIPPED",
+		[0x8]    = "LOCAL_SYMS_STRIPPED",
+		[0x10]   = "AGGRESSIVE_WS_TRIM",
+		[0x20]   = "LARGE_ADDRESS_AWARE",
+		[0x80]   = "BYTES_REVERSED_LO",
+		[0x100]  = "32BIT_MACHINE",
+		[0x200]  = "DEBUG_STRIPPED",
+		[0x400]  = "REMOVABLE_RUN_FROM_SWAP",
+		[0x800]  = "NET_RUN_FROM_SWAP",
+		[0x1000] = "SYSTEM",
+		[0x2000] = "DLL",
+		[0x4000] = "UP_SYSTEM_ONLY",
+		[0x8000] = "BYTES_REVERSED_HI"
+	} &default=function(i: count):string { return fmt("unknown-%d", i); };
+
+	const dll_characteristics: table[count] of string = {
+		[0x40]   = "DYNAMIC_BASE",
+		[0x80]   = "FORCE_INTEGRITY",
+		[0x100]  = "NX_COMPAT",
+		[0x200]  = "NO_ISOLATION",
+		[0x400]  = "NO_SEH",
+		[0x800]  = "NO_BIND",
+		[0x2000] = "WDM_DRIVER",
+		[0x8000] = "TERMINAL_SERVER_AWARE"
+	} &default=function(i: count):string { return fmt("unknown-%d", i); };
+
+	const windows_subsystems: table[count] of string = {
+		[0]  = "UNKNOWN",
+		[1]  = "NATIVE",
+		[2]  = "WINDOWS_GUI",
+		[3]  = "WINDOWS_CUI",
+		[7]  = "POSIX_CUI",
+		[9]  = "WINDOWS_CE_GUI",
+		[10] = "EFI_APPLICATION",
+		[11] = "EFI_BOOT_SERVICE_DRIVER",
+		[12] = "EFI_RUNTIME_
DRIVER",
+		[13] = "EFI_ROM",
+		[14] = "XBOX"
+	} &default=function(i: count):string { return fmt("unknown-%d", i); };
+
+	const directories: table[count] of string = {
+		[0]  = "Export Table",
+		[1]  = "Import Table",
+		[2]  = "Resource Table",
+		[3]  = "Exception Table",
+		[4]  = "Certificate Table",
+		[5]  = "Base Relocation Table",
+		[6]  = "Debug",
+		[7]  = "Architecture",
+		[8]  = "Global Ptr",
+		[9]  = "TLS Table",
+		[10] = "Load Config Table",
+		[11] = "Bound Import",
+		[12] = "IAT",
+		[13] = "Delay Import Descriptor",
+		[14] = "CLR Runtime Header",
+		[15] = "Reserved"
+	} &default=function(i: count):string { return fmt("unknown-%d", i); };
+
+	const section_characteristics: table[count] of string = {
+		[0x8]        = "TYPE_NO_PAD",
+		[0x20]       = "CNT_CODE",
+		[0x40]       = "CNT_INITIALIZED_DATA",
+		[0x80]       = "CNT_UNINITIALIZED_DATA",
+		[0x100]      = "LNK_OTHER",
+		[0x200]      = "LNK_INFO",
+		[0x800]      = "LNK_REMOVE",
+		[0x1000]     = "LNK_COMDAT",
+		[0x8000]     = "GPREL",
+		[0x20000]    = "MEM_16BIT",
+		[0x40000]    = "MEM_LOCKED",
+		[0x80000]    = "MEM_PRELOAD",
+		[0x100000]   = "ALIGN_1BYTES",
+		[0x200000]   = "ALIGN_2BYTES",
+		[0x300000]   = "ALIGN_4BYTES",
+		[0x400000]   = "ALIGN_8BYTES",
+		[0x500000]   = "ALIGN_16BYTES",
+		[0x600000]   = "ALIGN_32BYTES",
+		[0x700000]   = "ALIGN_64BYTES",
+		[0x800000]   = "ALIGN_128BYTES",
+		[0x900000]   = "ALIGN_256BYTES",
+		[0xa00000]   = "ALIGN_512BYTES",
+		[0xb00000]   = "ALIGN_1024BYTES",
+		[0xc00000]   = "ALIGN_2048BYTES",
+		[0xd00000]   = "ALIGN_4096BYTES",
+		[0xe00000]   = "ALIGN_8192BYTES",
+		[0x1000000]  = "LNK_NRELOC_OVFL",
+		[0x2000000]  = "MEM_DISCARDABLE",
+		[0x4000000]  = "MEM_NOT_CACHED",
+		[0x8000000]  = "MEM_NOT_PAGED",
+		[0x10000000] = "MEM_SHARED",
+		[0x20000000] = "MEM_EXECUTE",
+		[0x40000000] = "MEM_READ",
+		[0x80000000] = "MEM_WRITE"
+	} &default=function(i: count):string { return fmt("unknown-%d", i); };
+
+	const os_versions: table[count, count] of string = {
+		[10,0] = "Windows 10",
+		[6,4]  = "Windows 10 Technical Preview",
+		[6,3]  = "Windows 8.1 or Server 2012 R2",
+		[6,2]  = "Windows 8 or Server 2012",
+		[6,1]  = "Windows 7 or Server 2008 R2",
+		[6,0]  = "Windows Vista or Server 2008",
+		[5,2]  = "Windows XP x64 or Server 2003",
+		[5,1]  = "Windows XP",
+		[5,0]  = "Windows 2000",
+		[4,90] = "Windows Me",
+		[4,10] = "Windows 98",
+		[4,0]  = "Windows 95 or NT 4.0",
+		[3,51] = "Windows NT 3.51",
+		[3,50] = "Windows NT 3.5",
+		[3,2]  = "Windows 3.2",
+		[3,11] = "Windows for Workgroups 3.11",
+		[3,10] = "Windows 3.1 or NT 3.1",
+		[3,0]  = "Windows 3.0",
+		[2,11] = "Windows 2.11",
+		[2,10] = "Windows 2.10",
+		[2,0] = "Windows 2.0",
+		[1,4] = "Windows 1.04",
+		[1,3] = "Windows 1.03",
+		[1,1] = "Windows 1.01",
+		[1,0] = "Windows 1.0",
+	} &default=function(i: count, j: count):string { return fmt("unknown-%d.%d", i, j); };
+
+	const section_descs: table[string] of string = {
+		[".bss"]      = "Uninitialized data",
+		[".cormeta"]  = "CLR metadata that indicates that the object file contains managed code",
+		[".data"]     = "Initialized data",
+		[".debug$F"]  = "Generated FPO debug information",
+		[".debug$P"]  = "Precompiled debug types",
+		[".debug$S"]  = "Debug symbols",
+		[".debug$T"]  = "Debug types",
+		[".drective"] = "Linker options",
+		[".edata"]    = "Export tables",
+		[".idata"]    = "Import tables",
+		[".idlsym"]   = "Includes registered SEH to support IDL attributes",
+		[".pdata"]    = "Exception information",
+		[".rdata"]    = "Read-only initialized data",
+		[".reloc"]    = "Image relocations",
+		[".rsrc"]     = "Resource directory",
+		[".sbss"]     = "GP-relative uninitialized data",
+		[".sdata"]    = "GP-relative initialized data",
+		[".srdata"]   = "GP-relative read-only data",
+		[".sxdata"]   = "Registered exception handler data",
+		[".text"]     = "Executable code",
+		[".tls"]      = "Thread-local storage",
+		[".tls$"]     = "Thread-local storage",
+		[".vsdata"]   = "GP-relative initialized data",
+		[".xdata"]    = "Exception information",
+	} &default=function(i: string):string { return fmt("unknown-%s", i); };
+
+}

scripts/base/files/pe/main.bro

@@ -0,0 +1,137 @@
+module PE;
+
+@load ./consts.bro
+
+export {
+	redef enum Log::ID += { LOG };
+
+	type Info: record {
+		## Current timestamp.
+		ts:                  time              &log;
+		## File id of this portable executable file.
+		id:                  string            &log;
+		## The target machine that the file was compiled for.
+		machine:             string            &log &optional;
+		## The time that the file was created at.
+		compile_ts:          time              &log &optional;
+		## The required operating system.
+		os:                  string            &log &optional;
+		## The subsystem that is required to run this file.
+		subsystem:           string            &log &optional;
+		## Is the file an executable, or just an object file?
+		is_exe:              bool              &log &default=T;
+		## Is the file a 64-bit executable?
+		is_64bit:            bool              &log &default=T;
+		## Does the file support Address Space Layout Randomization?
+		uses_aslr:           bool              &log &default=F;
+		## Does the file support Data Execution Prevention?
+		uses_dep:            bool              &log &default=F;
+		## Does the file enforce code integrity checks?
+		uses_code_integrity: bool              &log &default=F;
+		## Does the file use structured exception handing?
+		uses_seh:            bool              &log &default=T;
+		## Does the file have an import table?
+		has_import_table:    bool              &log &optional;
+		## Does the file have an export table?
+		has_export_table:    bool              &log &optional;
+		## Does the file have an attribute certificate table?
+		has_cert_table:      bool              &log &optional;
+		## Does the file have a debug table?
+		has_debug_data:      bool              &log &optional;
+		## The names of the sections, in order.
+		section_names:       vector of string  &log &optional;
+	};
+
+	## Event for accessing logged records.
+	global log_pe: event(rec: Info);
+
+	## A hook that gets called when we first see a PE file.
+	global set_file: hook(f: fa_file);
+}
+
+redef record fa_file += {
+	pe: Info &optional;
+};
+
+const pe_mime_types = { "application/x-dosexec" };
+
+event bro_init() &priority=5
+	{
+	Files::register_for_mime_types(Files::ANALYZER_PE, pe_mime_types);
+	Log::create_stream(LOG, [$columns=Info, $ev=log_pe, $path="pe"]);
+	}
+
+hook set_file(f: fa_file) &priority=5
+	{
+	if ( ! f?$pe )
+		f$pe = [$ts=network_time(), $id=f$id];
+	}
+
+event pe_dos_header(f: fa_file, h: PE::DOSHeader) &priority=5
+	{
+	hook set_file(f);
+	}
+
+event pe_file_header(f: fa_file, h: PE::FileHeader) &priority=5
+	{
+	hook set_file(f);
+
+	f$pe$machine    = machine_types[h$machine];
+	f$pe$compile_ts = h$ts;
+	f$pe$is_exe     = ( h$optional_header_size > 0 );
+
+	for ( c in h$characteristics )
+		{
+		if ( file_characteristics[c] == "32BIT_MACHINE" )
+			f$pe$is_64bit = F;
+		}
+	}
+
+event pe_optional_header(f: fa_file, h: PE::OptionalHeader) &priority=5
+	{
+	hook set_file(f);
+
+	# Only EXEs have optional headers
+	if ( ! f$pe$is_exe )
+		return;
+
+	f$pe$os        = os_versions[h$os_version_major, h$os_version_minor];
+	f$pe$subsystem = windows_subsystems[h$subsystem];
+
+	for ( c in h$dll_characteristics )
+		{
+		if ( dll_characteristics[c] == "DYNAMIC_BASE" )
+			f$pe$uses_aslr = T;
+		if ( dll_characteristics[c] == "FORCE_INTEGRITY" )
+			f$pe$uses_code_integrity = T;
+		if ( dll_characteristics[c] == "NX_COMPAT" )
+			f$pe$uses_dep = T;
+		if ( dll_characteristics[c] == "NO_SEH" )
+			f$pe$uses_seh = F;
+		}
+
+	f$pe$has_export_table = (|h$table_sizes| > 0 && h$table_sizes[0] > 0);
+	f$pe$has_import_table = (|h$table_sizes| > 1 && h$table_sizes[1] > 0);
+	f$pe$has_cert_table   = (|h$table_sizes| > 4 && h$table_sizes[4] > 0);
+	f$pe$has_debug_data   = (|h$table_sizes| > 6 && h$table_sizes[6] > 0);
+	}
+
+event pe_section_header(f: fa_file, h: PE::SectionHeader) &priority=5
+	{
+	hook set_file(f);
+
+	# Only EXEs have section headers
+	if ( ! f$pe$is_exe )
+		return;
+
+	if ( ! f$pe?$section_names )
+		f$pe$section_names = vector();
+	f$pe$section_names[|f$pe$section_names|] = h$name;
+	}
+
+event file_state_remove(f: fa_file) &priority=-5
+	{
+	if ( f?$pe && f$pe?$machine )
+		Log::write(LOG, f$pe);
+	}
+

scripts/base/files/unified2/main.bro

@@ -71,11 +71,50 @@ global classification_map: table[count] of string;
 global sid_map: table[count] of string;
 global gen_map: table[count] of string;
 
+global num_classification_map_reads = 0;
+global num_sid_map_reads = 0;
+global num_gen_map_reads = 0;
+global watching = F;
+
 # For reading in config files.
 type OneLine: record {
 	line: string;
 };
 
+function mappings_initialized(): bool
+	{
+	return num_classification_map_reads > 0 &&
+	       num_sid_map_reads > 0 &&
+	       num_gen_map_reads > 0;
+	}
+
+function start_watching()
+	{
+	if ( watching )
+		return;
+
+	watching = T;
+
+	if ( watch_dir != "" )
+		{
+		Dir::monitor(watch_dir, function(fname: string)
+			{
+			Input::add_analysis([$source=fname,
+			                     $reader=Input::READER_BINARY,
+			                     $mode=Input::STREAM,
+			                     $name=fname]);
+			}, 10secs);
+		}
+
+	if ( watch_file != "" )
+		{
+		Input::add_analysis([$source=watch_file,
+		                     $reader=Input::READER_BINARY,
+		                     $mode=Input::STREAM,
+		                     $name=watch_file]);
+		}
+	}
+
 function create_info(ev: IDSEvent): Info
 	{
 	local info = Info($ts=ev$ts,
@@ -113,34 +152,56 @@ redef record fa_file += {
 
 event Unified2::read_sid_msg_line(desc: Input::EventDescription, tpe: Input::Event, line: string)
 	{
-	local parts = split_n(line, / \|\| /, F, 100);
-	if ( |parts| >= 2 && /^[0-9]+$/ in parts[1] )
-		sid_map[to_count(parts[1])] = parts[2];
+	local parts = split_string_n(line, / \|\| /, F, 100);
+	if ( |parts| >= 2 && /^[0-9]+$/ in parts[0] )
+		sid_map[to_count(parts[0])] = parts[1];
 	}
 
 event Unified2::read_gen_msg_line(desc: Input::EventDescription, tpe: Input::Event, line: string)
 	{
-	local parts = split_n(line, / \|\| /, F, 3);
-	if ( |parts| >= 2 && /^[0-9]+$/ in parts[1] )
-		gen_map[to_count(parts[1])] = parts[3];
+	local parts = split_string_n(line, / \|\| /, F, 3);
+	if ( |parts| >= 2 && /^[0-9]+$/ in parts[0] )
+		gen_map[to_count(parts[0])] = parts[2];
 	}
 
 event Unified2::read_classification_line(desc: Input::EventDescription, tpe: Input::Event, line: string)
 	{
-	local parts = split_n(line, /: /, F, 2);
+	local parts = split_string_n(line, /: /, F, 2);
 	if ( |parts| == 2 )
 		{
-		local parts2 = split_n(parts[2], /,/, F, 4);
+		local parts2 = split_string_n(parts[1], /,/, F, 4);
 		if ( |parts2| > 1 )
-			classification_map[|classification_map|+1] = parts2[1];
+			classification_map[|classification_map|+1] = parts2[0];
 		}
 	}
 
+event Input::end_of_data(name: string, source: string)
+	{
+	if ( name == classification_config )
+		++num_classification_map_reads;
+	else if ( name == sid_msg )
+		++num_sid_map_reads;
+	else if ( name == gen_msg )
+		++num_gen_map_reads;
+	else
+		return;
+
+	if ( watching )
+		return;
+
+	if ( mappings_initialized() )
+		start_watching();
+	}
+
 event bro_init() &priority=5
 	{
-	Log::create_stream(Unified2::LOG, [$columns=Info, $ev=log_unified2]);
+	Log::create_stream(Unified2::LOG, [$columns=Info, $ev=log_unified2, $path="unified2"]);
 
-	if ( sid_msg != "" )
+	if ( sid_msg == "" )
+		{
+		num_sid_map_reads = 1;
+		}
+	else
 		{
 		Input::add_event([$source=sid_msg,
 		                  $reader=Input::READER_RAW,
@@ -151,7 +212,11 @@ event bro_init() &priority=5
 		                  $ev=Unified2::read_sid_msg_line]);
 		}
 
-	if ( gen_msg != "" )
+	if ( gen_msg == "" )
+		{
+		num_gen_map_reads = 1;
+		}
+	else
 		{
 		Input::add_event([$source=gen_msg,
 		                  $name=gen_msg,
@@ -162,7 +227,11 @@ event bro_init() &priority=5
 		                  $ev=Unified2::read_gen_msg_line]);
 		}
 
-	if ( classification_config != "" )
+	if ( classification_config == "" )
+		{
+		num_classification_map_reads = 1;
+		}
+	else
 		{
 		Input::add_event([$source=classification_config,
 		                  $name=classification_config,
@@ -173,32 +242,16 @@ event bro_init() &priority=5
 		                  $ev=Unified2::read_classification_line]);
 		}
 
-	if ( watch_dir != "" )
-		{
-		Dir::monitor(watch_dir, function(fname: string)
-			{
-			Input::add_analysis([$source=fname,
-			                     $reader=Input::READER_BINARY,
-			                     $mode=Input::STREAM,
-			                     $name=fname]);
-			}, 10secs);
-		}
-
-	if ( watch_file != "" )
-		{
-		Input::add_analysis([$source=watch_file,
-		                     $reader=Input::READER_BINARY,
-		                     $mode=Input::STREAM,
-		                     $name=watch_file]);
-		}
+	if ( mappings_initialized() )
+		start_watching();
 	}
 
 event file_new(f: fa_file)
 	{
 	local file_dir = "";
-	local parts = split_all(f$source, /\/[^\/]*$/);
+	local parts = split_string_all(f$source, /\/[^\/]*$/);
 	if ( |parts| == 3 )
-		file_dir = parts[1];
+		file_dir = parts[0];
 
 	if ( (watch_file != "" && f$source == watch_file) || 
 	     (watch_dir != "" && compress_path(watch_dir) == file_dir) )

scripts/base/files/x509/main.bro

@@ -36,7 +36,7 @@ export {
 
 event bro_init() &priority=5
 	{
-	Log::create_stream(X509::LOG, [$columns=Info, $ev=log_x509]);
+	Log::create_stream(X509::LOG, [$columns=Info, $ev=log_x509, $path="x509"]);
 	}
 
 redef record Files::Info += {
@@ -47,6 +47,9 @@ redef record Files::Info += {
 
 event x509_certificate(f: fa_file, cert_ref: opaque of x509, cert: X509::Certificate) &priority=5
 	{
+	if ( ! f$info?$mime_type )
+		f$info$mime_type = "application/pkix-cert";
+
 	f$info$x509 = [$ts=f$info$ts, $id=f$id, $certificate=cert, $handle=cert_ref];
 	}
 

scripts/base/frameworks/broker/README

@@ -0,0 +1,2 @@
+The Broker communication framework facilitates connecting to remote Bro
+instances to share state and transfer events.

scripts/base/frameworks/broker/__load__.bro

@@ -0,0 +1 @@
+@load ./main

scripts/base/frameworks/broker/main.bro

@@ -0,0 +1,103 @@
+##! Various data structure definitions for use with Bro's communication system.
+
+module BrokerComm;
+
+export {
+
+	## A name used to identify this endpoint to peers.
+	## .. bro:see:: BrokerComm::connect BrokerComm::listen
+	const endpoint_name = "" &redef;
+
+	## Change communication behavior.
+	type EndpointFlags: record {
+		## Whether to restrict message topics that can be published to peers.
+		auto_publish: bool &default = T;
+		## Whether to restrict what message topics or data store identifiers
+		## the local endpoint advertises to peers (e.g. subscribing to
+		## events or making a master data store available).
+		auto_advertise: bool &default = T;
+	};
+
+	## Fine-grained tuning of communication behavior for a particular message.
+	type SendFlags: record {
+		## Send the message to the local endpoint.
+		self: bool &default = F;
+		## Send the message to peer endpoints that advertise interest in
+		## the topic associated with the message.
+		peers: bool &default = T;
+		## Send the message to peer endpoints even if they don't advertise
+		## interest in the topic associated with the message.
+		unsolicited: bool &default = F;
+	};
+
+	## Opaque communication data.
+	type Data: record {
+		d: opaque of BrokerComm::Data &optional;
+	};
+
+	## Opaque communication data.
+	type DataVector: vector of BrokerComm::Data;
+
+	## Opaque event communication data.
+	type EventArgs: record {
+		## The name of the event.  Not set if invalid event or arguments.
+		name: string &optional;
+		## The arguments to the event.
+		args: DataVector;
+	};
+
+	## Opaque communication data used as a convenient way to wrap key-value
+	## pairs that comprise table entries.
+	type TableItem : record {
+		key: BrokerComm::Data;
+		val: BrokerComm::Data;
+	};
+}
+
+module BrokerStore;
+
+export {
+
+	## Whether a data store query could be completed or not.
+	type QueryStatus: enum {
+		SUCCESS,
+		FAILURE,
+	};
+
+	## An expiry time for a key-value pair inserted in to a data store.
+	type ExpiryTime: record {
+		## Absolute point in time at which to expire the entry.
+		absolute: time &optional;
+		## A point in time relative to the last modification time at which
+		## to expire the entry.  New modifications will delay the expiration.
+		since_last_modification: interval &optional;
+	};
+
+	## The result of a data store query.
+	type QueryResult: record {
+		## Whether the query completed or not.
+		status: BrokerStore::QueryStatus;
+		## The result of the query.  Certain queries may use a particular
+		## data type (e.g. querying store size always returns a count, but
+		## a lookup may return various data types).
+		result: BrokerComm::Data;
+	};
+
+	## Options to tune the SQLite storage backend.
+	type SQLiteOptions: record {
+		## File system path of the database.
+		path: string &default = "store.sqlite";
+	};
+
+	## Options to tune the RocksDB storage backend.
+	type RocksDBOptions: record {
+		## File system path of the database.
+		path: string &default = "store.rocksdb";
+	};
+
+	## Options to tune the particular storage backends.
+	type BackendOptions: record {
+		sqlite: SQLiteOptions &default = SQLiteOptions();
+		rocksdb: RocksDBOptions &default = RocksDBOptions();
+	};
+}

scripts/base/frameworks/cluster/main.bro

@@ -43,35 +43,35 @@ export {
 		## software.
 		TIME_MACHINE,
 	};
-	
+
 	## Events raised by a manager and handled by the workers.
 	const manager2worker_events = /Drop::.*/ &redef;
-	
+
 	## Events raised by a manager and handled by proxies.
 	const manager2proxy_events = /EMPTY/ &redef;
-	
+
 	## Events raised by proxies and handled by a manager.
 	const proxy2manager_events = /EMPTY/ &redef;
-	
+
 	## Events raised by proxies and handled by workers.
 	const proxy2worker_events = /EMPTY/ &redef;
-	
+
 	## Events raised by workers and handled by a manager.
 	const worker2manager_events = /(TimeMachine::command|Drop::.*)/ &redef;
-	
+
 	## Events raised by workers and handled by proxies.
 	const worker2proxy_events = /EMPTY/ &redef;
-	
+
 	## Events raised by TimeMachine instances and handled by a manager.
 	const tm2manager_events = /EMPTY/ &redef;
-	
+
 	## Events raised by TimeMachine instances and handled by workers.
 	const tm2worker_events = /EMPTY/ &redef;
-	
-	## Events sent by the control host (i.e. BroControl) when dynamically 
+
+	## Events sent by the control host (i.e. BroControl) when dynamically
 	## connecting to a running instance to update settings or request data.
 	const control_events = Control::controller_events &redef;
-	
+
 	## Record type to indicate a node in a cluster.
 	type Node: record {
 		## Identifies the type of cluster node in this node's configuration.
@@ -96,13 +96,13 @@ export {
 		## Name of a time machine node with which this node connects.
 		time_machine: string      &optional;
 	};
-	
+
 	## This function can be called at any time to determine if the cluster
 	## framework is being enabled for this run.
 	##
 	## Returns: True if :bro:id:`Cluster::node` has been set.
 	global is_enabled: function(): bool;
-	
+
 	## This function can be called at any time to determine what type of
 	## cluster node the current Bro instance is going to be acting as.
 	## If :bro:id:`Cluster::is_enabled` returns false, then
@@ -110,22 +110,25 @@ export {
 	##
 	## Returns: The :bro:type:`Cluster::NodeType` the calling node acts as.
 	global local_node_type: function(): NodeType;
-	
+
 	## This gives the value for the number of workers currently connected to,
-	## and it's maintained internally by the cluster framework.  It's 
-	## primarily intended for use by managers to find out how many workers 
+	## and it's maintained internally by the cluster framework.  It's
+	## primarily intended for use by managers to find out how many workers
 	## should be responding to requests.
 	global worker_count: count = 0;
-	
+
 	## The cluster layout definition.  This should be placed into a filter
-	## named cluster-layout.bro somewhere in the BROPATH.  It will be 
+	## named cluster-layout.bro somewhere in the BROPATH.  It will be
 	## automatically loaded if the CLUSTER_NODE environment variable is set.
 	## Note that BroControl handles all of this automatically.
 	const nodes: table[string] of Node = {} &redef;
-	
+
 	## This is usually supplied on the command line for each instance
 	## of the cluster that is started up.
 	const node = getenv("CLUSTER_NODE") &redef;
+
+	## Interval for retrying failed connections between cluster nodes.
+	const retry_interval = 1min &redef;
 }
 
 function is_enabled(): bool
@@ -158,6 +161,6 @@ event bro_init() &priority=5
 		Reporter::error(fmt("'%s' is not a valid node in the Cluster::nodes configuration", node));
 		terminate();
 		}
-	
-	Log::create_stream(Cluster::LOG, [$columns=Info]);
+
+	Log::create_stream(Cluster::LOG, [$columns=Info, $path="cluster"]);
 	}

scripts/base/frameworks/cluster/setup-connections.bro

@@ -11,7 +11,7 @@ module Cluster;
 event bro_init() &priority=9
 	{
 	local me = nodes[node];
-	
+
 	for ( i in Cluster::nodes )
 		{
 		local n = nodes[i];
@@ -22,35 +22,35 @@ event bro_init() &priority=9
 			Communication::nodes["control"] = [$host=n$ip, $zone_id=n$zone_id,
 			                                   $connect=F, $class="control",
 			                                   $events=control_events];
-		
+
 		if ( me$node_type == MANAGER )
 			{
 			if ( n$node_type == WORKER && n$manager == node )
 				Communication::nodes[i] =
 				    [$host=n$ip, $zone_id=n$zone_id, $connect=F,
 				     $class=i, $events=worker2manager_events, $request_logs=T];
-			
+
 			if ( n$node_type == PROXY && n$manager == node )
 				Communication::nodes[i] =
 				    [$host=n$ip, $zone_id=n$zone_id, $connect=F,
 				     $class=i, $events=proxy2manager_events, $request_logs=T];
-				
+
 			if ( n$node_type == TIME_MACHINE && me?$time_machine && me$time_machine == i )
 				Communication::nodes["time-machine"] = [$host=nodes[i]$ip,
 				                                        $zone_id=nodes[i]$zone_id,
 				                                        $p=nodes[i]$p,
-				                                        $connect=T, $retry=1min,
+				                                        $connect=T, $retry=retry_interval,
 				                                        $events=tm2manager_events];
 			}
-		
+
 		else if ( me$node_type == PROXY )
 			{
 			if ( n$node_type == WORKER && n$proxy == node )
 				Communication::nodes[i] =
 					[$host=n$ip, $zone_id=n$zone_id, $connect=F, $class=i,
 					 $sync=T, $auth=T, $events=worker2proxy_events];
-			
-			# accepts connections from the previous one. 
+
+			# accepts connections from the previous one.
 			# (This is not ideal for setups with many proxies)
 			# FIXME: Once we're using multiple proxies, we should also figure out some $class scheme ...
 			if ( n$node_type == PROXY )
@@ -58,49 +58,49 @@ event bro_init() &priority=9
 				if ( n?$proxy )
 					Communication::nodes[i]
 					     = [$host=n$ip, $zone_id=n$zone_id, $p=n$p,
-					        $connect=T, $auth=F, $sync=T, $retry=1mins];
+					        $connect=T, $auth=F, $sync=T, $retry=retry_interval];
 				else if ( me?$proxy && me$proxy == i )
 					Communication::nodes[me$proxy]
 					     = [$host=nodes[i]$ip, $zone_id=nodes[i]$zone_id,
 					        $connect=F, $auth=T, $sync=T];
 				}
-			
+
 			# Finally the manager, to send it status updates.
 			if ( n$node_type == MANAGER && me$manager == i )
-				Communication::nodes["manager"] = [$host=nodes[i]$ip, 
-				                                   $zone_id=nodes[i]$zone_id, 
-				                                   $p=nodes[i]$p, 
-				                                   $connect=T, $retry=1mins, 
+				Communication::nodes["manager"] = [$host=nodes[i]$ip,
+				                                   $zone_id=nodes[i]$zone_id,
+				                                   $p=nodes[i]$p,
+				                                   $connect=T, $retry=retry_interval,
 				                                   $class=node,
 				                                   $events=manager2proxy_events];
 			}
 		else if ( me$node_type == WORKER )
 			{
 			if ( n$node_type == MANAGER && me$manager == i )
-				Communication::nodes["manager"] = [$host=nodes[i]$ip, 
+				Communication::nodes["manager"] = [$host=nodes[i]$ip,
 				                                   $zone_id=nodes[i]$zone_id,
 				                                   $p=nodes[i]$p,
-				                                   $connect=T, $retry=1mins, 
-				                                   $class=node, 
+				                                   $connect=T, $retry=retry_interval,
+				                                   $class=node,
 				                                   $events=manager2worker_events];
-			
+
 			if ( n$node_type == PROXY && me$proxy == i )
-				Communication::nodes["proxy"] = [$host=nodes[i]$ip, 
+				Communication::nodes["proxy"] = [$host=nodes[i]$ip,
 				                                 $zone_id=nodes[i]$zone_id,
 				                                 $p=nodes[i]$p,
-				                                 $connect=T, $retry=1mins, 
-				                                 $sync=T, $class=node, 
+				                                 $connect=T, $retry=retry_interval,
+				                                 $sync=T, $class=node,
 				                                 $events=proxy2worker_events];
-			
-			if ( n$node_type == TIME_MACHINE && 
+
+			if ( n$node_type == TIME_MACHINE &&
 			     me?$time_machine && me$time_machine == i )
-				Communication::nodes["time-machine"] = [$host=nodes[i]$ip, 
+				Communication::nodes["time-machine"] = [$host=nodes[i]$ip,
 				                                        $zone_id=nodes[i]$zone_id,
 				                                        $p=nodes[i]$p,
-				                                        $connect=T, 
-				                                        $retry=1min,
+				                                        $connect=T,
+				                                        $retry=retry_interval,
 				                                        $events=tm2worker_events];
-			
+
 			}
 		}
 	}

scripts/base/frameworks/communication/main.bro

@@ -164,7 +164,7 @@ const src_names = {
 
 event bro_init() &priority=5
 	{
-	Log::create_stream(Communication::LOG, [$columns=Info]);
+	Log::create_stream(Communication::LOG, [$columns=Info, $path="communication"]);
 	}
 
 function do_script_log_common(level: count, src: count, msg: string)

scripts/base/frameworks/dpd/main.bro

@@ -38,7 +38,7 @@ redef record connection += {
 
 event bro_init() &priority=5
 	{
-	Log::create_stream(DPD::LOG, [$columns=Info]);
+	Log::create_stream(DPD::LOG, [$columns=Info, $path="dpd"]);
 	}
 
 event protocol_confirmation(c: connection, atype: Analyzer::Tag, aid: count) &priority=10

scripts/base/frameworks/files/magic/__load__.bro

@@ -1,2 +1,9 @@
+@load-sigs ./archive
+@load-sigs ./audio
+@load-sigs ./font
 @load-sigs ./general
-@load-sigs ./libmagic
+@load-sigs ./image
+@load-sigs ./msoffice
+@load-sigs ./video
+
+@load-sigs ./libmagic

scripts/base/frameworks/files/magic/archive.sig

@@ -0,0 +1,176 @@
+
+signature file-tar {
+    file-magic /^[[:print:]\x00]{100}([[:digit:]\x20]{7}\x00){3}([[:digit:]\x20]{11}\x00){2}([[:digit:]\x00\x20]{7}[\x20\x00])[0-7\x00]/
+    file-mime "application/x-tar", 100
+}
+
+# This is low priority so that files using zip as a
+# container will be identified correctly.
+signature file-zip {
+	file-mime "application/zip", 10
+	file-magic /^PK\x03\x04.{2}/
+}
+
+# Multivolume Zip archive
+signature file-multi-zip {
+	file-mime "application/zip", 10
+	file-magic /^PK\x07\x08PK\x03\x04/
+}
+
+# RAR
+signature file-rar {
+	file-mime "application/x-rar", 70
+	file-magic /^Rar!/
+}
+
+# GZIP
+signature file-gzip {
+	file-mime "application/x-gzip", 100
+	file-magic /\x1f\x8b/
+}
+
+# Microsoft Cabinet
+signature file-ms-cab {
+	file-mime "application/vnd.ms-cab-compressed", 110
+	file-magic /^MSCF\x00\x00\x00\x00/
+}
+
+# Mac OS X DMG files
+signature file-dmg {
+	file-magic /^(\x78\x01\x73\x0D\x62\x62\x60|\x78\xDA\x63\x60\x18\x05|\x78\x01\x63\x60\x18\x05|\x78\xDA\x73\x0D|\x78[\x01\xDA]\xED[\xD0-\xD9])/
+	file-mime "application/x-dmg", 100
+}
+
+# XAR (eXtensible ARchive) format.
+# Mac OS X uses this for the .pkg format.
+signature file-xar {
+	file-magic /^xar\!/
+	file-mime "application/x-xar", 100
+}
+
+# RPM
+signature file-magic-auto352 {
+	file-mime "application/x-rpm", 70
+	file-magic /^(drpm|\xed\xab\xee\xdb)/
+}
+
+# StuffIt
+signature file-stuffit {
+	file-mime "application/x-stuffit", 70
+	file-magic /^(SIT\x21|StuffIt)/
+}
+
+# Archived data
+signature file-x-archive {
+	file-mime "application/x-archive", 70
+	file-magic /^!?<ar(ch)?>/
+}
+
+# ARC archive data
+signature file-arc {
+	file-mime "application/x-arc", 70
+	file-magic /^[\x00-\x7f]{2}[\x02-\x0a\x14\x48]\x1a/
+}
+
+# EET archive
+signature file-eet {
+	file-mime "application/x-eet", 70
+	file-magic /^\x1e\xe7\xff\x00/
+}
+
+# Zoo archive
+signature file-zoo {
+	file-mime "application/x-zoo", 70
+	file-magic /^.{20}\xdc\xa7\xc4\xfd/
+}
+
+# LZ4 compressed data (legacy format)
+signature file-lz4-legacy {
+	file-mime "application/x-lz4", 70
+	file-magic /(\x02\x21\x4c\x18)/
+}
+
+# LZ4 compressed data
+signature file-lz4 {
+	file-mime "application/x-lz4", 70
+	file-magic /^\x04\x22\x4d\x18/
+}
+
+# LRZIP compressed data
+signature file-lrzip {
+	file-mime "application/x-lrzip", 1
+	file-magic /^LRZI/
+}
+
+# LZIP compressed data
+signature file-lzip {
+	file-mime "application/x-lzip", 70
+	file-magic /^LZIP/
+}
+
+# Self-extracting PKZIP archive
+signature file-magic-auto434 {
+	file-mime "application/zip", 340
+	file-magic /^MZ.{28}(Copyright 1989\x2d1990 PKWARE Inc|PKLITE Copr)\x2e/
+}
+
+# LHA archive (LZH)
+signature file-lzh {
+	file-mime "application/x-lzh", 80
+	file-magic /^.{2}-(lh[ abcdex0-9]|lz[s2-8]|lz[s2-8]|pm[s012]|pc1)-/
+}
+
+# WARC Archive
+signature file-warc {
+	file-mime "application/warc", 50
+	file-magic /^WARC\x2f/
+}
+
+# 7-zip archive data
+signature file-7zip {
+	file-mime "application/x-7z-compressed", 50
+	file-magic /^7z\xbc\xaf\x27\x1c/
+}
+
+# XZ compressed data
+signature file-xz {
+	file-mime "application/x-xz", 90
+	file-magic /^\xfd7zXZ\x00/
+}
+
+# LHa self-extracting archive
+signature file-magic-auto436 {
+	file-mime "application/x-lha", 120
+	file-magic /^MZ.{34}LH[aA]\x27s SFX/
+}
+
+# ARJ archive data
+signature file-arj {
+	file-mime "application/x-arj", 50
+	file-magic /^\x60\xea/
+}
+
+# Byte-swapped cpio archive
+signature file-bs-cpio {
+	file-mime "application/x-cpio", 50
+	file-magic /(\x71\xc7|\xc7\x71)/
+}
+
+# CPIO archive
+signature file-cpio {
+	file-mime "application/x-cpio", 50
+	file-magic /^(\xc7\x71|\x71\xc7)/
+}
+
+# Compress'd data
+signature file-compress {
+	file-mime "application/x-compress", 50
+	file-magic /^\x1f\x9d/
+}
+
+# LZMA compressed data
+signature file-lzma {
+	file-mime "application/x-lzma", 71
+	file-magic /^\x5d\x00\x00/
+}
+

scripts/base/frameworks/files/magic/audio.sig

@@ -0,0 +1,13 @@
+
+# MPEG v3 audio
+signature file-mpeg-audio {
+	file-mime "audio/mpeg", 20
+	file-magic /^\xff[\xe2\xe3\xf2\xf3\xf6\xf7\xfa\xfb\xfc\xfd]/
+}
+
+# MPEG v4 audio
+signature file-m4a {
+	file-mime "audio/m4a", 70
+	file-magic /^....ftyp(m4a)/
+}
+

scripts/base/frameworks/files/magic/font.sig

@@ -0,0 +1,41 @@
+
+# Web Open Font Format
+signature file-woff {
+	file-magic /^wOFF/
+	file-mime "application/font-woff", 70
+}
+
+# TrueType font
+signature file-ttf {
+	file-mime "application/x-font-ttf", 80
+	file-magic /^\x00\x01\x00\x00\x00/
+}
+
+signature file-embedded-opentype {
+	file-mime "application/vnd.ms-fontobject", 50
+	file-magic /^.{34}LP/
+}
+
+# X11 SNF font
+signature file-snf {
+	file-mime "application/x-font-sfn", 70
+	file-magic /^(\x04\x00\x00\x00|\x00\x00\x00\x04).{100}(\x04\x00\x00\x00|\x00\x00\x00\x04)/
+}
+
+# OpenType font
+signature file-opentype {
+	file-mime "application/vnd.ms-opentype", 70
+	file-magic /^OTTO/
+}
+
+# FrameMaker Font file
+signature file-maker-screen-font {
+	file-mime "application/x-mif", 190
+	file-magic /^\x3cMakerScreenFont/
+}
+
+# >0  string,=SplineFontDB: (len=13), ["Spline Font Database "], swap_endian=0
+signature file-spline-font-db {
+	file-mime "application/vnd.font-fontforge-sfd", 160
+	file-magic /^SplineFontDB\x3a/
+}

scripts/base/frameworks/files/magic/general.sig

@@ -1,11 +1,272 @@
 # General purpose file magic signatures.
 
+# Plaintext
+# (Including BOMs for UTF-8, 16, and 32)
 signature file-plaintext {
-    file-magic /([[:print:][:space:]]{10})/
-    file-mime "text/plain", -20
+	file-mime "text/plain", -20
+	file-magic /^(\xef\xbb\xbf|(\x00\x00)?\xfe\xff|\xff\xfe(\x00\x00)?)?[[:space:]\x20-\x7E]{10}/
 }
 
-signature file-tar {
-    file-magic /([[:print:]\x00]){100}(([[:digit:]\x00\x20]){8}){3}/
-    file-mime "application/x-tar", 150
+signature file-json {
+	file-mime "text/json", 1
+	file-magic /^(\xef\xbb\xbf)?[\x0d\x0a[:blank:]]*\{[\x0d\x0a[:blank:]]*(["][^"]{1,}["]|[a-zA-Z][a-zA-Z0-9\\_]*)[\x0d\x0a[:blank:]]*:[\x0d\x0a[:blank:]]*(["]|\[|\{|[0-9]|true|false)/
+}
+
+signature file-json2 {
+	file-mime "text/json", 1
+	file-magic /^(\xef\xbb\xbf)?[\x0d\x0a[:blank:]]*\[[\x0d\x0a[:blank:]]*(((["][^"]{1,}["]|[0-9]{1,}(\.[0-9]{1,})?|true|false)[\x0d\x0a[:blank:]]*,)|\{|\[)[\x0d\x0a[:blank:]]*/
+}
+
+# Match empty JSON documents.
+signature file-json3 {
+	file-mime "text/json", 0
+	file-magic /^(\xef\xbb\xbf)?[\x0d\x0a[:blank:]]*(\[\]|\{\})[\x0d\x0a[:blank:]]*$/
+}
+
+signature file-xml {
+	file-mime "application/xml", 10
+	file-magic /^(\xef\xbb\xbf)?([\x0d\x0a[:blank:]]*(<!--.*-->)?[\x0d\x0a[:blank:]]*)*<\?xml /
+}
+
+signature file-xhtml {
+	file-mime "text/html", 100
+	file-magic /^(\xef\xbb\xbf)?([\x0d\x0a[:blank:]]*(<!--.*-->)?[\x0d\x0a[:blank:]]*)*(<\?xml .*\?>)?([\x0d\x0a[:blank:]]*(<!--.*-->)?[\x0d\x0a[:blank:]]*)*<(![dD][oO][cC][tT][yY][pP][eE] {1,}[hH][tT][mM][lL]|[hH][tT][mM][lL]|[mM][eE][tT][aA] {1,}[hH][tT][tT][pP]-[eE][qQ][uU][iI][vV])/
+}
+
+signature file-html {
+	file-mime "text/html", 49
+	file-magic /^(\xef\xbb\xbf)?([\x0d\x0a[:blank:]]*(<!--.*-->)?[\x0d\x0a[:blank:]]*)*(<\?xml .*\?>)?([\x0d\x0a[:blank:]]*(<!--.*-->)?[\x0d\x0a[:blank:]]*)*<![dD][oO][cC][tT][yY][pP][eE] {1,}[hH][tT][mM][lL]/
+}
+
+signature file-html2 {
+	file-mime "text/html", 20
+	file-magic /^(\xef\xbb\xbf)?([\x0d\x0a[:blank:]]*(<!--.*-->)?[\x0d\x0a[:blank:]]*)*(<\?xml .*\?>)?([\x0d\x0a[:blank:]]*(<!--.*-->)?[\x0d\x0a[:blank:]]*)*<([hH][eE][aA][dD]|[hH][tT][mM][lL]|[tT][iI][tT][lL][eE]|[bB][oO][dD][yY])/
+}
+
+signature file-rss {
+	file-mime "text/rss", 90
+	file-magic /^(\xef\xbb\xbf)?([\x0d\x0a[:blank:]]*(<!--.*-->)?[\x0d\x0a[:blank:]]*)*(<\?xml .*\?>)?([\x0d\x0a[:blank:]]*(<!--.*-->)?[\x0d\x0a[:blank:]]*)*<[rR][sS][sS]/
+}
+
+signature file-atom {
+	file-mime "text/atom", 100
+	file-magic /^(\xef\xbb\xbf)?([\x0d\x0a[:blank:]]*(<!--.*-->)?[\x0d\x0a[:blank:]]*)*(<\?xml .*\?>)?([\x0d\x0a[:blank:]]*(<!--.*-->)?[\x0d\x0a[:blank:]]*)*<([rR][sS][sS][^>]*xmlns:atom|[fF][eE][eE][dD][^>]*xmlns=["']?http:\/\/www.w3.org\/2005\/Atom["']?)/
+}
+
+signature file-soap {
+	file-mime "application/soap+xml", 49
+	file-magic /^(\xef\xbb\xbf)?([\x0d\x0a[:blank:]]*(<!--.*-->)?[\x0d\x0a[:blank:]]*)*(<\?xml .*\?>)?([\x0d\x0a[:blank:]]*(<!--.*-->)?[\x0d\x0a[:blank:]]*)*<[sS][oO][aA][pP](-[eE][nN][vV])?:[eE][nN][vV][eE][lL][oO][pP][eE]/
+}
+
+signature file-cross-domain-policy {
+	file-mime "text/x-cross-domain-policy", 49
+	file-magic /^([\x0d\x0a[:blank:]]*(<!--.*-->)?[\x0d\x0a[:blank:]]*)*(<\?xml .*\?>)?([\x0d\x0a[:blank:]]*(<!--.*-->)?[\x0d\x0a[:blank:]]*)*<![dD][oO][cC][tT][yY][pP][eE] {1,}[cC][rR][oO][sS][sS]-[dD][oO][mM][aA][iI][nN]-[pP][oO][lL][iI][cC][yY]/
+}
+
+signature file-cross-domain-policy2 {
+	file-mime "text/x-cross-domain-policy", 49
+	file-magic /^([\x0d\x0a[:blank:]]*(<!--.*-->)?[\x0d\x0a[:blank:]]*)*(<\?xml .*\?>)?([\x0d\x0a[:blank:]]*(<!--.*-->)?[\x0d\x0a[:blank:]]*)*<[cC][rR][oO][sS][sS]-[dD][oO][mM][aA][iI][nN]-[pP][oO][lL][iI][cC][yY]/
+}
+
+signature file-xmlrpc {
+	file-mime "application/xml-rpc", 49
+	file-magic /^(\xef\xbb\xbf)?([\x0d\x0a[:blank:]]*(<!--.*-->)?[\x0d\x0a[:blank:]]*)*(<\?xml .*\?>)?([\x0d\x0a[:blank:]]*(<!--.*-->)?[\x0d\x0a[:blank:]]*)*<[mM][eE][tT][hH][oO][dD][rR][eE][sS][pP][oO][nN][sS][eE]>/
+}
+
+signature file-coldfusion {
+	file-mime "magnus-internal/cold-fusion", 20
+	file-magic /^([\x0d\x0a[:blank:]]*(<!--.*-->)?)*<(CFPARAM|CFSET|CFIF)/
+}
+
+# Adobe Flash Media Manifest
+signature file-f4m {
+	file-mime "application/f4m", 49
+	file-magic /^(\xef\xbb\xbf)?([\x0d\x0a[:blank:]]*(<!--.*-->)?[\x0d\x0a[:blank:]]*)*(<\?xml .*\?>)?([\x0d\x0a[:blank:]]*(<!--.*-->)?[\x0d\x0a[:blank:]]*)*<[mM][aA][nN][iI][fF][eE][sS][tT][\x0d\x0a[:blank:]]{1,}xmlns=\"http:\/\/ns\.adobe\.com\/f4m\//
+}
+
+# Microsoft LNK files
+signature file-lnk {
+	file-mime "application/x-ms-shortcut", 49
+	file-magic /^\x4C\x00\x00\x00\x01\x14\x02\x00\x00\x00\x00\x00\xC0\x00\x00\x00\x00\x10\x00\x00\x00\x46/
+}
+
+signature file-jar {
+	file-mime "application/java-archive", 100
+	file-magic /^PK\x03\x04.{1,200}\x14\x00..META-INF\/MANIFEST\.MF/
+}
+
+signature file-java-applet {
+	file-mime "application/x-java-applet", 71
+	file-magic /^\xca\xfe\xba\xbe...[\x2d-\x34]/
+}
+
+# OCSP requests over HTTP.
+signature file-ocsp-request {
+	file-magic /^.{11,19}\x06\x05\x2b\x0e\x03\x02\x1a/
+	file-mime "application/ocsp-request", 71
+}
+
+# OCSP responses over HTTP.
+signature file-ocsp-response {
+	file-magic /^.{11,19}\x06\x09\x2B\x06\x01\x05\x05\x07\x30\x01\x01/
+	file-mime "application/ocsp-response", 71
+}
+
+# Shockwave flash
+signature file-swf {
+	file-magic /^(F|C|Z)WS/
+	file-mime "application/x-shockwave-flash", 60
+}
+
+# Microsoft Outlook's Transport Neutral Encapsulation Format
+signature file-tnef {
+	file-magic /^\x78\x9f\x3e\x22/
+	file-mime "application/vnd.ms-tnef", 100
+}
+
+# Mac OS X Mach-O executable
+signature file-mach-o {
+	file-magic /^[\xce\xcf]\xfa\xed\xfe/
+	file-mime "application/x-mach-o-executable", 100
+}
+
+# Mac OS X Universal Mach-O executable
+signature file-mach-o-universal {
+	file-magic /^\xca\xfe\xba\xbe..\x00[\x01-\x14]/
+	file-mime "application/x-mach-o-executable", 100
+}
+
+signature file-pkcs7 {
+	file-magic /^MIME-Version:.*protocol=\"application\/pkcs7-signature\"/
+	file-mime "application/pkcs7-signature", 100
+}
+
+# Concatenated X.509 certificates in textual format.
+signature file-pem {
+	file-magic /^-----BEGIN CERTIFICATE-----/
+	file-mime "application/x-pem"
+}
+
+# Java Web Start file.
+signature file-jnlp {
+	file-magic /^\<jnlp\x20/
+	file-mime "application/x-java-jnlp-file", 100
+}
+
+signature file-pcap {
+	file-magic /^(\xa1\xb2\xc3\xd4|\xd4\xc3\xb2\xa1)/
+	file-mime "application/vnd.tcpdump.pcap", 70
+}
+
+signature file-pcap-ng {
+	file-magic /^\x0a\x0d\x0d\x0a.{4}(\x1a\x2b\x3c\x4d|\x4d\x3c\x2b\x1a)/
+	file-mime "application/vnd.tcpdump.pcap", 100
+}
+
+signature file-shellscript {
+	file-mime "text/x-shellscript", 250
+	file-magic /^\x23\x21[^\n]{1,15}bin\/(env[[:space:]]+)?(ba|tc|c|z|fa|ae|k)?sh/
+}
+
+signature file-perl {
+	file-magic /^\x23\x21[^\n]{1,15}bin\/(env[[:space:]]+)?perl/
+	file-mime "text/x-perl", 60
+}
+
+signature file-ruby {
+	file-magic /^\x23\x21[^\n]{1,15}bin\/(env[[:space:]]+)?ruby/
+	file-mime "text/x-ruby", 60
+}
+
+signature file-python {
+	file-magic /^\x23\x21[^\n]{1,15}bin\/(env[[:space:]]+)?python/
+	file-mime "text/x-python", 60
+}
+
+signature file-awk {
+	file-mime "text/x-awk", 60
+	file-magic /^\x23\x21[^\n]{1,15}bin\/(env[[:space:]]+)?(g|n)?awk/
+}
+
+signature file-tcl {
+	file-mime "text/x-tcl", 60
+	file-magic /^\x23\x21[^\n]{1,15}bin\/(env[[:space:]]+)?(wish|tcl)/
+}
+
+signature file-lua {
+	file-mime "text/x-lua", 49
+	file-magic /^\x23\x21[^\n]{1,15}bin\/(env[[:space:]]+)?lua/
+}
+
+signature file-javascript {
+	file-mime "application/javascript", 60
+	file-magic /^\x23\x21[^\n]{1,15}bin\/(env[[:space:]]+)?node(js)?/
+}
+
+signature file-javascript2 {
+	file-mime "application/javascript", 60
+	file-magic /^[\x0d\x0a[:blank:]]*<[sS][cC][rR][iI][pP][tT][[:blank:]]+([tT][yY][pP][eE]|[lL][aA][nN][gG][uU][aA][gG][eE])=['"]?([tT][eE][xX][tT]\/)?[jJ][aA][vV][aA][sS][cC][rR][iI][pP][tT]/
+}
+
+signature file-javascript3 {
+	file-mime "application/javascript", 60
+	# This seems to be a somewhat common idiom in javascript.
+	file-magic /^[\x0d\x0a[:blank:]]*for \(;;\);/
+}
+
+signature file-javascript4 {
+	file-mime "application/javascript", 60
+	file-magic /^[\x0d\x0a[:blank:]]*document\.write(ln)?[:blank:]?\(/
+}
+
+signature file-javascript5 {
+	file-mime "application/javascript", 60
+	file-magic /^\(function\(\)[[:blank:]\n]*\{/
+}
+
+signature file-javascript6 {
+	file-mime "application/javascript", 60
+	file-magic /^[\x0d\x0a[:blank:]]*<script>[\x0d\x0a[:blank:]]*(var|function) /
+}
+
+signature file-php {
+	file-mime "text/x-php", 60
+	file-magic /^\x23\x21[^\n]{1,15}bin\/(env[[:space:]]+)?php/
+}
+
+signature file-php2 {
+	file-magic /^.*<\?php/
+	file-mime "text/x-php", 40
+}
+
+# Stereolithography ASCII format
+signature file-stl-ascii {
+	file-magic /^solid\x20/
+	file-mime "application/sla", 10
+}
+
+# Sketchup model file
+signature file-skp {
+	file-magic /^\xFF\xFE\xFF\x0E\x53\x00\x6B\x00\x65\x00\x74\x00\x63\x00\x68\x00\x55\x00\x70\x00\x20\x00\x4D\x00\x6F\x00\x64\x00\x65\x00\x6C\x00/
+	file-mime "application/skp", 100
+}
+
+signature file-elf-object {
+	file-mime "application/x-object", 50
+	file-magic /\x7fELF[\x01\x02](\x01.{10}\x01\x00|\x02.{10}\x00\x01)/
+}
+
+signature file-elf {
+	file-mime "application/x-executable", 50
+	file-magic /\x7fELF[\x01\x02](\x01.{10}\x02\x00|\x02.{10}\x00\x02)/
+}
+
+signature file-elf-sharedlib {
+	file-mime "application/x-sharedlib", 50
+	file-magic /\x7fELF[\x01\x02](\x01.{10}\x03\x00|\x02.{10}\x00\x03)/
+}
+
+signature file-elf-coredump {
+	file-mime "application/x-coredump", 50
+	file-magic /\x7fELF[\x01\x02](\x01.{10}\x04\x00|\x02.{10}\x00\x04)/
 }

scripts/base/frameworks/files/magic/image.sig

@@ -0,0 +1,166 @@
+
+signature file-tiff {
+	file-mime "image/tiff", 70
+	file-magic /^(MM\x00[\x2a\x2b]|II[\x2a\x2b]\x00)/
+}
+
+signature file-gif {
+	file-mime "image/gif", 70
+	file-magic /^GIF8/
+}
+
+# JPEG image
+signature file-jpeg {
+	file-mime "image/jpeg", 52
+	file-magic /^\xff\xd8/
+}
+
+signature file-bmp {
+	file-mime "image/x-ms-bmp", 50
+	file-magic /BM.{12}[\x0c\x28\x40\x6c\x7c\x80]\x00/
+}
+
+signature file-ico {
+	file-magic /^\x00\x00\x01\x00/
+	file-mime "image/x-icon", 70
+}
+
+signature file-cur {
+	file-magic /^\x00\x00\x02\x00/
+	file-mime "image/x-cursor", 70
+}
+
+signature file-magic-auto289 {
+	file-mime "image/vnd.adobe.photoshop", 70
+	file-magic /^8BPS/
+}
+
+signature file-png {
+	file-mime "image/png", 110
+	file-magic /^\x89PNG/
+}
+
+# JPEG 2000
+signature file-jp2 {
+	file-mime "image/jp2", 60
+	file-magic /.{4}ftypjp2/
+}
+
+# JPEG 2000
+signature file-jp22 {
+	file-mime "image/jp2", 70
+	file-magic /\x00\x00\x00\x0cjP  \x0d\x0a\x87\x0a.{8}jp2 /
+}
+
+# JPEG 2000
+signature file-jpx {
+	file-mime "image/jpx", 70
+	file-magic /\x00\x00\x00\x0cjP  \x0d\x0a\x87\x0a.{8}jpx /
+}
+
+# JPEG 2000
+signature file-jpm {
+	file-mime "image/jpm", 70
+	file-magic /\x00\x00\x00\x0cjP  \x0d\x0a\x87\x0a.{8}jpm /
+}
+
+# Xcursor image
+signature file-x-cursor {
+	file-mime "image/x-xcursor", 70
+	file-magic /^Xcur/
+}
+
+# NIFF image
+signature file-niff {
+	file-mime "image/x-niff", 70
+	file-magic /^IIN1/
+}
+
+# OpenEXR image
+signature file-openexr {
+	file-mime "image/x-exr", 70
+	file-magic /^\x76\x2f\x31\x01/
+}
+
+# DPX image
+signature file-dpx {
+	file-mime "image/x-dpx", 70
+	file-magic /^SDPX/
+}
+
+# Cartesian Perceptual Compression image
+signature file-cpi {
+	file-mime "image/x-cpi", 70
+	file-magic /(CPC\xb2)/
+}
+
+signature file-orf {
+	file-mime "image/x-olympus-orf", 70
+	file-magic /IIR[OS]|MMOR/
+}
+
+# Foveon X3F raw image
+signature file-x3r {
+	file-mime "image/x-x3f", 70
+	file-magic /^FOVb/
+}
+
+# Paint.NET image
+signature file-paint-net {
+	file-mime "image/x-paintnet", 70
+	file-magic /^PDN3/
+}
+
+# Corel Draw Picture
+signature file-coreldraw {
+	file-mime "image/x-coreldraw", 70
+	file-magic /^RIFF....CDR[A6]/
+}
+
+# Netpbm PAM image
+signature file-netbpm{
+	file-mime "image/x-portable-pixmap", 50
+	file-magic /^P7/
+}
+
+# JPEG 2000 image
+signature file-jpeg-2000 {
+	file-mime "image/jp2", 50
+	file-magic /^....jP/
+}
+
+# DjVU Images
+signature file-djvu {
+	file-mime "image/vnd.djvu", 70
+	file-magic /AT\x26TFORM.{4}(DJV[MUI]|THUM)/
+}
+
+# DWG AutoDesk AutoCAD
+signature file-dwg {
+	file-mime "image/vnd.dwg", 90
+	file-magic /^(AC[12]\.|AC10)/
+}
+
+# GIMP XCF image
+signature file-gimp-xcf {
+	file-mime "image/x-xcf", 110
+	file-magic /^gimp xcf/
+}
+
+# Polar Monitor Bitmap text
+signature file-polar-monitor-bitmap {
+	file-mime "image/x-polar-monitor-bitmap", 160
+	file-magic /^\x5bBitmapInfo2\x5d/
+}
+
+# Award BIOS bitmap
+signature file-award-bitmap {
+	file-mime "image/x-award-bmp", 20
+	file-magic /^AWBM/
+}
+
+# Award BIOS Logo, 136 x 84
+signature file-award-bios-logo {
+	file-mime "image/x-award-bioslogo", 50
+	file-magic /^\x11[\x06\x09]/
+}

scripts/base/frameworks/files/magic/msoffice.sig

@@ -0,0 +1,34 @@
+
+# This signature is non-specific and terrible but after
+# searching for a long time there doesn't seem to be a 
+# better option.  
+signature file-msword {
+	file-magic /^\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1/
+	file-mime "application/msword", 50
+}
+
+signature file-ooxml {
+	file-magic /^PK\x03\x04\x14\x00\x06\x00/
+	file-mime "application/vnd.openxmlformats-officedocument", 50
+}
+
+signature file-docx {
+	file-magic /^PK\x03\x04.{26}(\[Content_Types\]\.xml|_rels\x2f\.rels|word\x2f).*PK\x03\x04.{26}word\x2f/
+	file-mime "application/vnd.openxmlformats-officedocument.wordprocessingml.document", 80
+}
+
+signature file-xlsx {
+	file-magic /^PK\x03\x04.{26}(\[Content_Types\]\.xml|_rels\x2f\.rels|xl\2f).*PK\x03\x04.{26}xl\x2f/
+	file-mime "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", 80
+}
+
+signature file-pptx {
+	file-magic /^PK\x03\x04.{26}(\[Content_Types\]\.xml|_rels\x2f\.rels|ppt\x2f).*PK\x03\x04.{26}ppt\x2f/
+	file-mime "application/vnd.openxmlformats-officedocument.presentationml.presentation", 80
+}
+
+signature file-msaccess {
+	file-mime "application/x-msaccess", 180
+	file-magic /.{4}Standard (Jet|ACE) DB\x00/
+}
+

scripts/base/frameworks/files/magic/video.sig

@@ -0,0 +1,105 @@
+
+# Macromedia Flash Video
+signature file-flv {
+	file-mime "video/x-flv", 60
+	file-magic /^FLV/
+}
+
+# FLI animation
+signature file-fli {
+	file-mime "video/x-fli", 50
+	file-magic /^.{4}\x11\xaf/
+}
+
+# FLC animation
+signature file-flc {
+	file-mime "video/x-flc", 50
+	file-magic /^.{4}\x12\xaf/
+}
+
+# Motion JPEG 2000
+signature file-mj2 {
+	file-mime "video/mj2", 70
+	file-magic /\x00\x00\x00\x0cjP  \x0d\x0a\x87\x0a.{8}mjp2/
+}
+
+# MNG video
+signature file-mng {
+	file-mime "video/x-mng", 70
+	file-magic /^\x8aMNG/
+}
+
+# JNG video
+signature file-jng {
+	file-mime "video/x-jng", 70
+	file-magic /^\x8bJNG/
+}
+
+# Generic MPEG container
+signature file-mpeg {
+	file-mime "video/mpeg", 50
+	file-magic /(\x00\x00\x01[\xb0-\xbb])/
+}
+
+# MPV
+signature file-mpv {
+	file-mime "video/mpv", 71
+	file-magic /(\x00\x00\x01\xb3)/
+}
+
+# H.264
+signature file-h264 {
+	file-mime "video/h264", 41
+	file-magic /(\x00\x00\x00\x01)([\x07\x27\x47\x67\x87\xa7\xc7\xe7])/
+}
+
+# WebM video
+signature file-webm {
+	file-mime "video/webm", 70
+	file-magic /(\x1a\x45\xdf\xa3)(.*)(B\x82)(.{1})(webm)/
+}
+
+# Matroska video
+signature file-matroska {
+	file-mime "video/x-matroska", 110
+	file-magic /(\x1a\x45\xdf\xa3)(.*)(B\x82)(.{1})(matroska)/
+}
+
+# MP2P
+signature file-mp2p {
+	file-mime "video/mp2p", 21
+	file-magic /\x00\x00\x01\xba([\x40-\x7f\xc0-\xff])/
+}
+
+# MPEG transport stream data. These files typically have the extension "ts".
+# Note: The 0x47 repeats every 188 bytes. Using four as the number of
+# occurrences for the test here is arbitrary.
+signature file-mp2t {
+	file-mime "video/mp2t", 40
+	file-magic /^(\x47.{187}){4}/
+}
+
+# Silicon Graphics video
+signature file-sgi-movie {
+	file-mime "video/x-sgi-movie", 70
+	file-magic /^MOVI/
+}
+
+# Apple QuickTime movie
+signature file-quicktime {
+	file-mime "video/quicktime", 70
+	file-magic /^....(mdat|moov)/
+}
+
+# MPEG v4 video
+signature file-mp4 {
+	file-mime "video/mp4", 70
+	file-magic /^....ftyp(isom|mp4[12])/
+}
+
+# 3GPP Video
+signature file-3gpp {
+	file-mime "video/3gpp", 60
+	file-magic /^....ftyp(3g[egps2]|avc1|mmp4)/
+}
+

scripts/base/frameworks/files/main.bro

@@ -100,8 +100,9 @@ export {
 		## during the process of analysis e.g. due to dropped packets.
 		missing_bytes: count &log &default=0;
 
-		## The number of not all-in-sequence bytes in the file stream that
-		## were delivered to file analyzers due to reassembly buffer overflow.
+		## The number of bytes in the file stream that were not delivered to
+		## stream file analyzers.  This could be overlapping bytes or 
+		## bytes that couldn't be reassembled.
 		overflow_bytes: count &log &default=0;
 
 		## Whether the file analysis timed out at least once for the file.
@@ -124,6 +125,36 @@ export {
 	## generate two handles that would hash to the same file id.
 	const salt = "I recommend changing this." &redef;
 
+	## Decide if you want to automatically attached analyzers to 
+	## files based on the detected mime type of the file.
+	const analyze_by_mime_type_automatically = T &redef;
+
+	## The default setting for file reassembly.
+	const enable_reassembler = T &redef;
+
+	## The default per-file reassembly buffer size.
+	const reassembly_buffer_size = 524288 &redef;
+
+	## Allows the file reassembler to be used if it's necessary because the
+	## file is transferred out of order.
+	##
+	## f: the file.
+	global enable_reassembly: function(f: fa_file);
+
+	## Disables the file reassembler on this file.  If the file is not 
+	## transferred out of order this will have no effect.
+	##
+	## f: the file.
+	global disable_reassembly: function(f: fa_file);
+
+	## Set the maximum size the reassembly buffer is allowed to grow
+	## for the given file.
+	##
+	## f: the file.
+	##
+	## max: Maximum allowed size of the reassembly buffer.
+	global set_reassembly_buffer_size: function(f: fa_file, max: count);
+
 	## Sets the *timeout_interval* field of :bro:see:`fa_file`, which is
 	## used to determine the length of inactivity that is allowed for a file
 	## before internal state related to it is cleaned up.  When used within
@@ -153,15 +184,6 @@ export {
 	                              tag: Files::Tag,
 	                              args: AnalyzerArgs &default=AnalyzerArgs()): bool;
 
-	## Adds all analyzers associated with a give MIME type to the analysis of
-	## a file.  Note that analyzers added via MIME types cannot take further
-	## arguments.
-	##
-	## f: the file.
-	##
-	## mtype: the MIME type; it will be compared case-insensitive.
-	global add_analyzers_for_mime_type: function(f: fa_file, mtype: string);
-
 	## Removes an analyzer from the analysis of a given file.
 	##
 	## f: the file.
@@ -244,7 +266,7 @@ export {
 	## mts: The set of MIME types, each in the form "foo/bar" (case-insensitive).
 	##
 	## Returns: True if the MIME types were successfully registered.
-	global register_for_mime_types: function(tag: Analyzer::Tag, mts: set[string]) : bool;
+	global register_for_mime_types: function(tag: Files::Tag, mts: set[string]) : bool;
 
 	## Registers a MIME type for an analyzer. If a future file with this type is seen,
 	## the analyzer will be automatically assigned to parsing it. The function *adds*
@@ -255,20 +277,20 @@ export {
 	## mt: The MIME type in the form "foo/bar" (case-insensitive).
 	##
 	## Returns: True if the MIME type was successfully registered.
-	global register_for_mime_type: function(tag: Analyzer::Tag, mt: string) : bool;
+	global register_for_mime_type: function(tag: Files::Tag, mt: string) : bool;
 
 	## Returns a set of all MIME types currently registered for a specific analyzer.
 	##
 	## tag: The tag of the analyzer.
 	##
 	## Returns: The set of MIME types.
-	global registered_mime_types: function(tag: Analyzer::Tag) : set[string];
+	global registered_mime_types: function(tag: Files::Tag) : set[string];
 
 	## Returns a table of all MIME-type-to-analyzer mappings currently registered.
 	##
 	## Returns: A table mapping each analyzer to the set of MIME types
 	##          registered for it.
-	global all_registered_mime_types: function() : table[Analyzer::Tag] of set[string];
+	global all_registered_mime_types: function() : table[Files::Tag] of set[string];
 
 	## Event that can be handled to access the Info record as it is sent on
 	## to the logging framework.
@@ -283,13 +305,14 @@ redef record fa_file += {
 global registered_protocols: table[Analyzer::Tag] of ProtoRegistration = table();
 
 # Store the MIME type to analyzer mappings.
-global mime_types: table[Analyzer::Tag] of set[string];
+global mime_types: table[Files::Tag] of set[string];
+global mime_type_to_analyzers: table[string] of set[Files::Tag];
 
 global analyzer_add_callbacks: table[Files::Tag] of function(f: fa_file, args: AnalyzerArgs) = table();
 
 event bro_init() &priority=5
 	{
-	Log::create_stream(Files::LOG, [$columns=Info, $ev=log_files]);
+	Log::create_stream(Files::LOG, [$columns=Info, $ev=log_files, $path="files"]);
 	}
 
 function set_info(f: fa_file)
@@ -313,8 +336,6 @@ function set_info(f: fa_file)
 	f$info$overflow_bytes = f$overflow_bytes;
 	if ( f?$is_orig )
 		f$info$is_orig = f$is_orig;
-	if ( f?$mime_type )
-		f$info$mime_type = f$mime_type;
 	}
 
 function set_timeout_interval(f: fa_file, t: interval): bool
@@ -322,6 +343,21 @@ function set_timeout_interval(f: fa_file, t: interval): bool
 	return __set_timeout_interval(f$id, t);
 	}
 
+function enable_reassembly(f: fa_file)
+	{
+	__enable_reassembly(f$id);
+	}
+
+function disable_reassembly(f: fa_file)
+	{
+	__disable_reassembly(f$id);
+	}
+
+function set_reassembly_buffer_size(f: fa_file, max: count)
+	{
+	__set_reassembly_buffer(f$id, max);
+	}
+
 function add_analyzer(f: fa_file, tag: Files::Tag, args: AnalyzerArgs): bool
 	{
 	add f$info$analyzers[Files::analyzer_name(tag)];
@@ -337,15 +373,6 @@ function add_analyzer(f: fa_file, tag: Files::Tag, args: AnalyzerArgs): bool
 	return T;
 	}
 
-function add_analyzers_for_mime_type(f: fa_file, mtype: string)
-	{
-	local dummy_args: AnalyzerArgs;
-	local analyzers = __add_analyzers_for_mime_type(f$id, mtype, dummy_args);
-
-	for ( tag in analyzers )
-		add f$info$analyzers[Files::analyzer_name(tag)];
-	}
-
 function register_analyzer_add_callback(tag: Files::Tag, callback: function(f: fa_file, args: AnalyzerArgs))
 	{
 	analyzer_add_callbacks[tag] = callback;
@@ -366,17 +393,87 @@ function analyzer_name(tag: Files::Tag): string
 	return __analyzer_name(tag);
 	}
 
+function register_protocol(tag: Analyzer::Tag, reg: ProtoRegistration): bool
+	{
+	local result = (tag !in registered_protocols);
+	registered_protocols[tag] = reg;
+	return result;
+	}
+
+function register_for_mime_types(tag: Files::Tag, mime_types: set[string]) : bool
+	{
+	local rc = T;
+
+	for ( mt in mime_types )
+		{
+		if ( ! register_for_mime_type(tag, mt) )
+			rc = F;
+		}
+
+	return rc;
+	}
+
+function register_for_mime_type(tag: Files::Tag, mt: string) : bool
+	{
+	if ( tag !in mime_types )
+		{
+		mime_types[tag] = set();
+		}
+	add mime_types[tag][mt];
+
+	if ( mt !in mime_type_to_analyzers )
+		{
+		mime_type_to_analyzers[mt] = set();
+		}
+	add mime_type_to_analyzers[mt][tag];
+
+	return T;
+	}
+
+function registered_mime_types(tag: Files::Tag) : set[string]
+	{
+	return tag in mime_types ? mime_types[tag] : set();
+	}
+
+function all_registered_mime_types(): table[Files::Tag] of set[string]
+	{
+	return mime_types;
+	}
+
+function describe(f: fa_file): string
+	{
+	local tag = Analyzer::get_tag(f$source);
+	if ( tag !in registered_protocols )
+		return "";
+
+	local handler = registered_protocols[tag];
+	return handler$describe(f);
+	}
+
+event get_file_handle(tag: Files::Tag, c: connection, is_orig: bool) &priority=5
+	{
+	if ( tag !in registered_protocols )
+		return;
+
+	local handler = registered_protocols[tag];
+	set_file_handle(handler$get_file_handle(c, is_orig));
+	}
+
 event file_new(f: fa_file) &priority=10
 	{
 	set_info(f);
 
-	if ( f?$mime_type )
-		add_analyzers_for_mime_type(f, f$mime_type);
+	if ( enable_reassembler )
+		{
+		Files::enable_reassembly(f);
+		Files::set_reassembly_buffer_size(f, reassembly_buffer_size);
+		}
 	}
 
 event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priority=10
 	{
 	set_info(f);
+
 	add f$info$conn_uids[c$uid];
 	local cid = c$id;
 	add f$info$tx_hosts[f$is_orig ? cid$orig_h : cid$resp_h];
@@ -386,6 +483,27 @@ event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priori
 	add f$info$rx_hosts[f$is_orig ? cid$resp_h : cid$orig_h];
 	}
 
+event file_sniff(f: fa_file, meta: fa_metadata) &priority=10
+	{
+	set_info(f);
+
+	if ( ! meta?$mime_type )
+		return;
+
+	f$info$mime_type = meta$mime_type;
+
+	if ( analyze_by_mime_type_automatically &&
+	     meta$mime_type in mime_type_to_analyzers )
+		{
+		local analyzers = mime_type_to_analyzers[meta$mime_type];
+		for ( a in analyzers )
+			{
+			add f$info$analyzers[Files::analyzer_name(a)];
+			Files::add_analyzer(f, a);
+			}
+		}
+	}
+
 event file_timeout(f: fa_file) &priority=10
 	{
 	set_info(f);
@@ -401,64 +519,3 @@ event file_state_remove(f: fa_file) &priority=-10
 	{
 	Log::write(Files::LOG, f$info);
 	}
-
-function register_protocol(tag: Analyzer::Tag, reg: ProtoRegistration): bool
-	{
-	local result = (tag !in registered_protocols);
-	registered_protocols[tag] = reg;
-	return result;
-	}
-
-function register_for_mime_types(tag: Analyzer::Tag, mime_types: set[string]) : bool
-	{
-	local rc = T;
-
-	for ( mt in mime_types )
-		{
-		if ( ! register_for_mime_type(tag, mt) )
-			rc = F;
-		}
-
-	return rc;
-	}
-
-function register_for_mime_type(tag: Analyzer::Tag, mt: string) : bool
-	{
-	if ( ! __register_for_mime_type(tag, mt) )
-		return F;
-
-	if ( tag !in mime_types )
-		mime_types[tag] = set();
-
-	add mime_types[tag][mt];
-	return T;
-	}
-
-function registered_mime_types(tag: Analyzer::Tag) : set[string]
-	{
-	return tag in mime_types ? mime_types[tag] : set();
-	}
-
-function all_registered_mime_types(): table[Analyzer::Tag] of set[string]
-	{
-	return mime_types;
-	}
-
-function describe(f: fa_file): string
-	{
-	local tag = Analyzer::get_tag(f$source);
-	if ( tag !in registered_protocols )
-		return "";
-
-	local handler = registered_protocols[tag];
-	return handler$describe(f);
-	}
-
-event get_file_handle(tag: Analyzer::Tag, c: connection, is_orig: bool) &priority=5
-	{
-	if ( tag !in registered_protocols )
-		return;
-
-	local handler = registered_protocols[tag];
-	set_file_handle(handler$get_file_handle(c, is_orig));
-	}

scripts/base/frameworks/input/main.bro

@@ -1,18 +1,25 @@
 ##! The input framework provides a way to read previously stored data either
-##! as an event stream or into a bro table.
+##! as an event stream or into a Bro table.
 
 module Input;
 
 export {
 	type Event: enum {
+		## New data has been imported.
 		EVENT_NEW = 0,
+		## Existing data has been changed.
 		EVENT_CHANGED = 1,
+		## Previously existing data has been removed.
 		EVENT_REMOVED = 2,
 	};
 
+	## Type that defines the input stream read mode.
 	type Mode: enum {
+		## Do not automatically reread the file after it has been read.
 		MANUAL = 0,
+		## Reread the entire file each time a change is found.
 		REREAD = 1,
+		## Read data from end of file each time new data is appended.
 		STREAM = 2
 	};
 
@@ -24,20 +31,20 @@ export {
 
 	## Separator between fields.
 	## Please note that the separator has to be exactly one character long.
-	## Can be overwritten by individual writers.
+	## Individual readers can use a different value.
 	const separator = "\t" &redef;
 
 	## Separator between set elements.
 	## Please note that the separator has to be exactly one character long.
-	## Can be overwritten by individual writers.
+	## Individual readers can use a different value.
 	const set_separator = "," &redef;
 
 	## String to use for empty fields.
-	## Can be overwritten by individual writers.
+	## Individual readers can use a different value.
 	const empty_field = "(empty)" &redef;
 
 	## String to use for an unset &optional field.
-	## Can be overwritten by individual writers.
+	## Individual readers can use a different value.
 	const unset_field = "-" &redef;
 
 	## Flag that controls if the input framework accepts records
@@ -47,11 +54,11 @@ export {
 	## abort. Defaults to false (abort).
 	const accept_unsupported_types = F &redef;
 
-	## TableFilter description type used for the `table` method.
+	## A table input stream type used to send data to a Bro table.
 	type TableDescription: record {
 		# Common definitions for tables and events
 
-		## String that allows the reader to find the source.
+		## String that allows the reader to find the source of the data.
 		## For `READER_ASCII`, this is the filename.
 		source: string;
 
@@ -61,7 +68,8 @@ export {
 		## Read mode to use for this stream.
 		mode: Mode &default=default_mode;
 
-		## Descriptive name. Used to remove a stream at a later time.
+		## Name of the input stream.  This is used by some functions to
+		## manipulate the stream.
 		name: string;
 
 		# Special definitions for tables
@@ -73,31 +81,35 @@ export {
 		idx: any;
 
 		## Record that defines the values used as the elements of the table.
-		## If this is undefined, then *destination* has to be a set.
+		## If this is undefined, then *destination* must be a set.
 		val: any &optional;
 
-		## Defines if the value of the table is a record (default), or a single value.
-		## When this is set to false, then *val* can only contain one element.
+		## Defines if the value of the table is a record (default), or a single
+		## value. When this is set to false, then *val* can only contain one
+		## element.
 		want_record: bool &default=T;
 
-		## The event that is raised each time a value is added to, changed in or removed
-		## from the table. The event will receive an Input::Event enum as the first
-		## argument, the *idx* record as the second argument and the value (record) as the
-		## third argument.
-		ev: any &optional; # event containing idx, val as values.
+		## The event that is raised each time a value is added to, changed in,
+		## or removed from the table. The event will receive an
+		## Input::TableDescription as the first argument, an Input::Event
+		## enum as the second argument, the *idx* record as the third argument
+		## and the value (record) as the fourth argument.
+		ev: any &optional;
 
-		## Predicate function that can decide if an insertion, update or removal should
-		## really be executed. Parameters are the same as for the event. If true is
-		## returned, the update is performed. If false is returned, it is skipped.
+		## Predicate function that can decide if an insertion, update or removal
+		## should really be executed. Parameters have same meaning as for the
+		## event.
+		## If true is returned, the update is performed. If false is returned,
+		## it is skipped.
 		pred: function(typ: Input::Event, left: any, right: any): bool &optional;
 
-		## A key/value table that will be passed on the reader.
-		## Interpretation of the values is left to the writer, but
+		## A key/value table that will be passed to the reader.
+		## Interpretation of the values is left to the reader, but
 		## usually they will be used for configuration purposes.
-                config: table[string] of string &default=table();
+		config: table[string] of string &default=table();
 	};
 
-	## EventFilter description type used for the `event` method.
+	## An event input stream type used to send input data to a Bro event.
 	type EventDescription: record {
 		# Common definitions for tables and events
 
@@ -116,19 +128,26 @@ export {
 
 		# Special definitions for events
 
-		## Record describing the fields to be retrieved from the source input.
+		## Record type describing the fields to be retrieved from the input
+		## source.
 		fields: any;
 
-		## If this is false, the event receives each value in fields as a separate argument.
-		## If this is set to true (default), the event receives all fields in a single record value.
+		## If this is false, the event receives each value in *fields* as a
+		## separate argument.
+		## If this is set to true (default), the event receives all fields in
+		## a single record value.
 		want_record: bool &default=T;
 
-		## The event that is raised each time a new line is received from the reader.
-		## The event will receive an Input::Event enum as the first element, and the fields as the following arguments.
+		## The event that is raised each time a new line is received from the
+		## reader. The event will receive an Input::EventDescription record
+		## as the first argument, an Input::Event enum as the second
+		## argument, and the fields (as specified in *fields*) as the following
+		## arguments (this will either be a single record value containing
+		## all fields, or each field value as a separate argument).
 		ev: any;
 
-		## A key/value table that will be passed on the reader.
-		## Interpretation of the values is left to the writer, but
+		## A key/value table that will be passed to the reader.
+		## Interpretation of the values is left to the reader, but
 		## usually they will be used for configuration purposes.
 		config: table[string] of string &default=table();
 	};
@@ -155,28 +174,29 @@ export {
 		## field will be the same value as the *source* field.
 		name: string;
 
-		## A key/value table that will be passed on the reader.
-		## Interpretation of the values is left to the writer, but
+		## A key/value table that will be passed to the reader.
+		## Interpretation of the values is left to the reader, but
 		## usually they will be used for configuration purposes.
 		config: table[string] of string &default=table();
 	};
 
-	## Create a new table input from a given source.
+	## Create a new table input stream from a given source.
 	##
 	## description: `TableDescription` record describing the source.
 	##
 	## Returns: true on success.
 	global add_table: function(description: Input::TableDescription) : bool;
 
-	## Create a new event input from a given source.
+	## Create a new event input stream from a given source.
 	##
 	## description: `EventDescription` record describing the source.
 	##
 	## Returns: true on success.
 	global add_event: function(description: Input::EventDescription) : bool;
 
-	## Create a new file analysis input from a given source.  Data read from
-	## the source is automatically forwarded to the file analysis framework.
+	## Create a new file analysis input stream from a given source.  Data read
+	## from the source is automatically forwarded to the file analysis
+	## framework.
 	##
 	## description: A record describing the source.
 	##
@@ -199,7 +219,11 @@ export {
 
 	## Event that is called when the end of a data source has been reached,
 	## including after an update.
-	global end_of_data: event(name: string, source:string);
+	##
+	## name: Name of the input stream.
+	##
+	## source: String that identifies the data source (such as the filename).
+	global end_of_data: event(name: string, source: string);
 }
 
 @load base/bif/input.bif

scripts/base/frameworks/input/readers/raw.bro

@@ -11,7 +11,9 @@ export {
 	##
 	## name: name of the input stream.
 	## source: source of the input stream.
-	## exit_code: exit code of the program, or number of the signal that forced the program to exit.
-	## signal_exit: false when program exited normally, true when program was forced to exit by a signal.
+	## exit_code: exit code of the program, or number of the signal that forced
+	##            the program to exit.
+	## signal_exit: false when program exited normally, true when program was
+	##              forced to exit by a signal.
 	global process_finished: event(name: string, source:string, exit_code:count, signal_exit:bool);
 }

scripts/base/frameworks/intel/main.bro

@@ -32,6 +32,8 @@ export {
 		FILE_NAME,
 		## Certificate SHA-1 hash.
 		CERT_HASH,
+		## Public key MD5 hash. (SSH server host keys are a good example.)
+		PUBKEY_HASH,
 	};
 	
 	## Data about an :bro:type:`Intel::Item`.
@@ -67,6 +69,7 @@ export {
 		IN_ANYWHERE,
 	};
 
+	## Information about a piece of "seen" data.
 	type Seen: record {
 		## The string if the data is about a string.
 		indicator:       string        &log &optional;
@@ -124,7 +127,7 @@ export {
 		sources:  set[string]    &log &default=string_set();
 	};
 
-	## Intelligence data manipulation functions.
+	## Intelligence data manipulation function.
 	global insert: function(item: Item);
 
 	## Function to declare discovery of a piece of data in order to check
@@ -173,7 +176,7 @@ global min_data_store: MinDataStore &redef;
 
 event bro_init() &priority=5
 	{
-	Log::create_stream(LOG, [$columns=Info, $ev=log_intel]);
+	Log::create_stream(LOG, [$columns=Info, $ev=log_intel, $path="intel"]);
 	}
 
 function find(s: Seen): bool
@@ -289,8 +292,8 @@ event Intel::match(s: Seen, items: set[Item]) &priority=5
 		if ( ! info?$fuid )
 			info$fuid = s$f$id;
 
-		if ( ! info?$file_mime_type && s$f?$mime_type )
-			info$file_mime_type = s$f$mime_type;
+		if ( ! info?$file_mime_type && s$f?$info && s$f$info?$mime_type )
+			info$file_mime_type = s$f$info$mime_type;
 
 		if ( ! info?$file_desc )
 			info$file_desc = Files::describe(s$f);

scripts/base/frameworks/logging/main.bro

@@ -6,9 +6,10 @@
 module Log;
 
 export {
-	## Type that defines an ID unique to each log stream. Scripts creating new log
-	## streams need to redef this enum to add their own specific log ID. The log ID
-	## implicitly determines the default name of the generated log file.
+	## Type that defines an ID unique to each log stream. Scripts creating new
+	## log streams need to redef this enum to add their own specific log ID.
+	## The log ID implicitly determines the default name of the generated log
+	## file.
 	type Log::ID: enum {
 		## Dummy place-holder.
 		UNKNOWN
@@ -20,25 +21,24 @@ export {
 	## If true, remote logging is by default enabled for all filters.
 	const enable_remote_logging = T &redef;
 
-	## Default writer to use if a filter does not specify
-	## anything else.
+	## Default writer to use if a filter does not specify anything else.
 	const default_writer = WRITER_ASCII &redef;
 
-	## Default separator between fields for logwriters.
-	## Can be overwritten by individual writers.
+	## Default separator to use between fields.
+	## Individual writers can use a different value.
 	const separator = "\t" &redef;
 
-	## Separator between set elements.
-	## Can be overwritten by individual writers.
+	## Default separator to use between elements of a set.
+	## Individual writers can use a different value.
 	const set_separator = "," &redef;
 
-	## String to use for empty fields. This should be different from
-	## *unset_field* to make the output unambiguous.
-	## Can be overwritten by individual writers.
+	## Default string to use for empty fields. This should be different
+	## from *unset_field* to make the output unambiguous.
+	## Individual writers can use a different value.
 	const empty_field = "(empty)" &redef;
 
-	## String to use for an unset &optional field.
-	## Can be overwritten by individual writers.
+	## Default string to use for an unset &optional field.
+	## Individual writers can use a different value.
 	const unset_field = "-" &redef;
 
 	## Type defining the content of a logging stream.
@@ -50,11 +50,17 @@ export {
 		## The event receives a single same parameter, an instance of
 		## type ``columns``.
 		ev: any &optional;
+
+		## A path that will be inherited by any filters added to the
+		## stream which do not already specify their own path.
+		path: string &optional;
 	};
 
 	## Builds the default path values for log filters if not otherwise
 	## specified by a filter. The default implementation uses *id*
-	## to derive a name.
+	## to derive a name.  Upon adding a filter to a stream, if neither
+	## ``path`` nor ``path_func`` is explicitly set by them, then
+	## this function is used as the ``path_func``.
 	##
 	## id: The ID associated with the log stream.
 	##
@@ -63,7 +69,7 @@ export {
 	##       If no ``path`` is defined for the filter, then the first call
 	##       to the function will contain an empty string.
 	##
-	## rec: An instance of the streams's ``columns`` type with its
+	## rec: An instance of the stream's ``columns`` type with its
 	##      fields set to the values to be logged.
 	##
 	## Returns: The path to be used for the filter.
@@ -81,7 +87,8 @@ export {
 		terminating: bool;	##< True if rotation occured due to Bro shutting down.
 	};
 
-	## Default rotation interval. Zero disables rotation.
+	## Default rotation interval to use for filters that do not specify
+	## an interval. Zero disables rotation.
 	##
 	## Note that this is overridden by the BroControl LogRotationInterval
 	## option.
@@ -116,8 +123,8 @@ export {
 		## Indicates whether a log entry should be recorded.
 		## If not given, all entries are recorded.
 		##
-		## rec: An instance of the streams's ``columns`` type with its
-		##      fields set to the values to logged.
+		## rec: An instance of the stream's ``columns`` type with its
+		##      fields set to the values to be logged.
 		##
 		## Returns: True if the entry is to be recorded.
 		pred: function(rec: any): bool &optional;
@@ -125,10 +132,10 @@ export {
 		## Output path for recording entries matching this
 		## filter.
 		##
-		## The specific interpretation of the string is up to
-		## the used writer, and may for example be the destination
+		## The specific interpretation of the string is up to the
+		## logging writer, and may for example be the destination
 		## file name. Generally, filenames are expected to be given
-		## without any extensions; writers will add appropiate
+		## without any extensions; writers will add appropriate
 		## extensions automatically.
 		##
 		## If this path is found to conflict with another filter's
@@ -143,7 +150,9 @@ export {
 		## to compute the string dynamically. It is ok to return
 		## different strings for separate calls, but be careful: it's
 		## easy to flood the disk by returning a new string for each
-		## connection.
+		## connection.  Upon adding a filter to a stream, if neither
+		## ``path`` nor ``path_func`` is explicitly set by them, then
+		## :bro:see:`Log::default_path_func` is used.
 		##
 		## id: The ID associated with the log stream.
 		##
@@ -153,7 +162,7 @@ export {
 		##       then the first call to the function will contain an
 		##       empty string.
 		##
-		## rec: An instance of the streams's ``columns`` type with its
+		## rec: An instance of the stream's ``columns`` type with its
 		##      fields set to the values to be logged.
 		##
 		## Returns: The path to be used for the filter, which will be
@@ -177,7 +186,7 @@ export {
 		## If true, entries are passed on to remote peers.
 		log_remote: bool &default=enable_remote_logging;
 
-		## Rotation interval.
+		## Rotation interval. Zero disables rotation.
 		interv: interval &default=default_rotation_interval;
 
 		## Callback function to trigger for rotated files. If not set, the
@@ -207,9 +216,9 @@ export {
 
 	## Removes a logging stream completely, stopping all the threads.
 	##
-	## id: The ID enum to be associated with the new logging stream.
+	## id: The ID associated with the logging stream.
 	##
-	## Returns: True if a new stream was successfully removed.
+	## Returns: True if the stream was successfully removed.
 	##
 	## .. bro:see:: Log::create_stream
 	global remove_stream: function(id: ID) : bool;
@@ -379,6 +388,8 @@ export {
 	global active_streams: table[ID] of Stream = table();
 }
 
+global all_streams: table[ID] of Stream = table();
+
 # We keep a script-level copy of all filters so that we can manipulate them.
 global filters: table[ID, string] of Filter;
 
@@ -405,30 +416,30 @@ function default_path_func(id: ID, path: string, rec: any) : string
 
 	local id_str = fmt("%s", id);
 
-	local parts = split1(id_str, /::/);
+	local parts = split_string1(id_str, /::/);
 	if ( |parts| == 2 )
 		{
 		# Example: Notice::LOG -> "notice"
-		if ( parts[2] == "LOG" )
+		if ( parts[1] == "LOG" )
 			{
-			local module_parts = split_n(parts[1], /[^A-Z][A-Z][a-z]*/, T, 4);
+			local module_parts = split_string_n(parts[0], /[^A-Z][A-Z][a-z]*/, T, 4);
 			local output = "";
-			if ( 1 in module_parts )
-				output = module_parts[1];
+			if ( 0 in module_parts )
+				output = module_parts[0];
+			if ( 1 in module_parts && module_parts[1] != "" )
+				output = cat(output, sub_bytes(module_parts[1],1,1), "_", sub_bytes(module_parts[1], 2, |module_parts[1]|));
 			if ( 2 in module_parts && module_parts[2] != "" )
-				output = cat(output, sub_bytes(module_parts[2],1,1), "_", sub_bytes(module_parts[2], 2, |module_parts[2]|));
+				output = cat(output, "_", module_parts[2]);
 			if ( 3 in module_parts && module_parts[3] != "" )
-				output = cat(output, "_", module_parts[3]);
-			if ( 4 in module_parts && module_parts[4] != "" )
-				output = cat(output, sub_bytes(module_parts[4],1,1), "_", sub_bytes(module_parts[4], 2, |module_parts[4]|));
+				output = cat(output, sub_bytes(module_parts[3],1,1), "_", sub_bytes(module_parts[3], 2, |module_parts[3]|));
 			return to_lower(output);
 			}
 
 		# Example: Notice::POLICY_LOG -> "notice_policy"
-		if ( /_LOG$/ in parts[2] )
-			parts[2] = sub(parts[2], /_LOG$/, "");
+		if ( /_LOG$/ in parts[1] )
+			parts[1] = sub(parts[1], /_LOG$/, "");
 
-		return cat(to_lower(parts[1]),"_",to_lower(parts[2]));
+		return cat(to_lower(parts[0]),"_",to_lower(parts[1]));
 		}
 	else
 		return to_lower(id_str);
@@ -463,6 +474,7 @@ function create_stream(id: ID, stream: Stream) : bool
 		return F;
 
 	active_streams[id] = stream;
+	all_streams[id] = stream;
 
 	return add_default_filter(id);
 	}
@@ -470,6 +482,7 @@ function create_stream(id: ID, stream: Stream) : bool
 function remove_stream(id: ID) : bool
 	{
 	delete active_streams[id];
+	delete all_streams[id];
 	return __remove_stream(id);
 	}
 
@@ -482,10 +495,12 @@ function disable_stream(id: ID) : bool
 
 function add_filter(id: ID, filter: Filter) : bool
 	{
-	# This is a work-around for the fact that we can't forward-declare
-	# the default_path_func and then use it as &default in the record
-	# definition.
-	if ( ! filter?$path_func )
+	local stream = all_streams[id];
+
+	if ( stream?$path && ! filter?$path )
+		filter$path = stream$path;
+
+	if ( ! filter?$path && ! filter?$path_func )
 		filter$path_func = default_path_func;
 
 	filters[id, filter$name] = filter;

scripts/base/frameworks/logging/postprocessors/sftp.bro

@@ -37,6 +37,8 @@ export {
 		user: string;
 		## The remote host to which to transfer logs.
 		host: string;
+		## The port to connect to. Defaults to 22
+		host_port: count &default=22;
 		## The path/directory on the remote host to send logs.
 		path: string;
 	};
@@ -63,8 +65,8 @@ function sftp_postprocessor(info: Log::RotationInfo): bool
 		{
 		local dst = fmt("%s/%s.%s.log", d$path, info$path,
 		                strftime(Log::sftp_rotation_date_format, info$open));
-		command += fmt("echo put %s %s | sftp -b - %s@%s;", info$fname, dst,
-		               d$user, d$host);
+		command += fmt("echo put %s %s | sftp -P %d -b - %s@%s;", info$fname, dst,
+		               d$host_port, d$user, d$host);
 		}
 
 	command += fmt("/bin/rm %s", info$fname);

scripts/base/frameworks/logging/writers/ascii.bro

@@ -1,15 +1,15 @@
 ##! Interface for the ASCII log writer.  Redefinable options are available
 ##! to tweak the output format of ASCII logs.
 ##!
-##! The ASCII writer supports currently one writer-specific filter option via
-##! ``config``: setting ``tsv`` to the string ``T`` turns the output into
+##! The ASCII writer currently supports one writer-specific per-filter config
+##! option: setting ``tsv`` to the string ``T`` turns the output into
 ##! "tab-separated-value" mode where only a single header row with the column
 ##! names is printed out as meta information, with no "# fields" prepended; no
-##! other meta data gets included in that mode.
+##! other meta data gets included in that mode.  Example filter using this::
 ##!
-##! Example filter using this::
-##!
-##!    local my_filter: Log::Filter = [$name = "my-filter", $writer = Log::WRITER_ASCII, $config = table(["tsv"] = "T")];
+##!    local f: Log::Filter = [$name = "my-filter",
+##!                            $writer = Log::WRITER_ASCII,
+##!                            $config = table(["tsv"] = "T")];
 ##!
 
 module LogAscii;
@@ -29,6 +29,8 @@ export {
 	## Format of timestamps when writing out JSON. By default, the JSON
 	## formatter will use double values for timestamps which represent the
 	## number of seconds from the UNIX epoch.
+	##
+	## This option is also available as a per-filter ``$config`` option.
 	const json_timestamps: JSON::TimestampFormat = JSON::TS_EPOCH &redef;
 
 	## If true, include lines with log meta information such as column names

scripts/base/frameworks/logging/writers/sqlite.bro

@@ -19,7 +19,7 @@ export {
 	const unset_field = Log::unset_field &redef;
 
 	## String to use for empty fields. This should be different from
-        ## *unset_field* to make the output unambiguous.
+	## *unset_field* to make the output unambiguous.
 	const empty_field = Log::empty_field &redef;
 }
 

scripts/base/frameworks/notice/main.bro

@@ -19,9 +19,9 @@ export {
 	## the :bro:id:`NOTICE` function.  The convention is to give a general
 	## category along with the specific notice separating words with
 	## underscores and using leading capitals on each word except for
-	## abbreviations which are kept in all capitals.  For example,
+	## abbreviations which are kept in all capitals. For example,
 	## SSH::Password_Guessing is for hosts that have crossed a threshold of
-	## heuristically determined failed SSH logins.
+	## failed SSH logins.
 	type Type: enum {
 		## Notice reporting a count of how often a notice occurred.
 		Tally,
@@ -349,9 +349,9 @@ function log_mailing_postprocessor(info: Log::RotationInfo): bool
 
 event bro_init() &priority=5
 	{
-	Log::create_stream(Notice::LOG, [$columns=Info, $ev=log_notice]);
+	Log::create_stream(Notice::LOG, [$columns=Info, $ev=log_notice, $path="notice"]);
 
-	Log::create_stream(Notice::ALARM_LOG, [$columns=Notice::Info]);
+	Log::create_stream(Notice::ALARM_LOG, [$columns=Notice::Info, $path="notice_alarm"]);
 	# If Bro is configured for mailing notices, set up mailing for alarms.
 	# Make sure that this alarm log is also output as text so that it can
 	# be packaged up and emailed later.
@@ -531,8 +531,8 @@ function create_file_info(f: fa_file): Notice::FileInfo
 	local fi: Notice::FileInfo = Notice::FileInfo($fuid = f$id,
 	                                              $desc = Files::describe(f));
 
-	if ( f?$mime_type )
-		fi$mime = f$mime_type;
+	if ( f?$info && f$info?$mime_type )
+		fi$mime = f$info$mime_type;
 
 	if ( f?$conns && |f$conns| == 1 )
 		for ( id in f$conns )

scripts/base/frameworks/notice/weird.bro

@@ -294,7 +294,7 @@ global current_conn: connection;
 
 event bro_init() &priority=5
 	{
-	Log::create_stream(Weird::LOG, [$columns=Info, $ev=log_weird]);
+	Log::create_stream(Weird::LOG, [$columns=Info, $ev=log_weird, $path="weird"]);
 	}
 
 function flow_id_string(src: addr, dst: addr): string

scripts/base/frameworks/packet-filter/main.bro

@@ -138,7 +138,7 @@ redef enum PcapFilterID += {
 
 function test_filter(filter: string): bool
 	{
-	if ( ! precompile_pcap_filter(FilterTester, filter) )
+	if ( ! Pcap::precompile_pcap_filter(FilterTester, filter) )
 		{
 		# The given filter was invalid
 		# TODO: generate a notice.
@@ -159,7 +159,7 @@ event filter_change_tracking()
 
 event bro_init() &priority=5
 	{
-	Log::create_stream(PacketFilter::LOG, [$columns=Info]);
+	Log::create_stream(PacketFilter::LOG, [$columns=Info, $path="packet_filter"]);
 
 	# Preverify the capture and restrict filters to give more granular failure messages.
 	for ( id in capture_filters )
@@ -273,7 +273,7 @@ function install(): bool
 		return F;
 
 	local ts = current_time();
-	if ( ! precompile_pcap_filter(DefaultPcapFilter, tmp_filter) )
+	if ( ! Pcap::precompile_pcap_filter(DefaultPcapFilter, tmp_filter) )
 		{
 		NOTICE([$note=Compile_Failure,
 		        $msg=fmt("Compiling packet filter failed"),
@@ -303,7 +303,7 @@ function install(): bool
 		}
 	info$filter = current_filter;
 
-	if ( ! install_pcap_filter(DefaultPcapFilter) )
+	if ( ! Pcap::install_pcap_filter(DefaultPcapFilter) )
 		{
 		# Installing the filter failed for some reason.
 		info$success = F;

scripts/base/frameworks/reporter/main.bro

@@ -45,7 +45,7 @@ export {
 
 event bro_init() &priority=5
 	{
-	Log::create_stream(Reporter::LOG, [$columns=Info]);
+	Log::create_stream(Reporter::LOG, [$columns=Info, $path="reporter"]);
 	}
 
 event reporter_info(t: time, msg: string, location: string) &priority=-5

scripts/base/frameworks/signatures/main.bro

@@ -142,7 +142,7 @@ global did_sig_log: set[string] &read_expire = 1 hr;
 
 event bro_init()
 	{
-	Log::create_stream(Signatures::LOG, [$columns=Info, $ev=log_signature]);
+	Log::create_stream(Signatures::LOG, [$columns=Info, $ev=log_signature, $path="signatures"]);
 	}
 		
 # Returns true if the given signature has already been triggered for the given
@@ -277,7 +277,7 @@ event signature_match(state: signature_state, msg: string, data: string)
 				orig, sig_id, hcount);
 
 		Log::write(Signatures::LOG, 
-			[$note=Multiple_Sig_Responders,
+			[$ts=network_time(), $note=Multiple_Sig_Responders,
 		     $src_addr=orig, $sig_id=sig_id, $event_msg=msg,
 		     $host_count=hcount, $sub_msg=horz_scan_msg]);
 

scripts/base/frameworks/software/main.bro

@@ -105,7 +105,7 @@ export {
 
 event bro_init() &priority=5
 	{
-	Log::create_stream(Software::LOG, [$columns=Info, $ev=log_software]);
+	Log::create_stream(Software::LOG, [$columns=Info, $ev=log_software, $path="software"]);
 	}
 	
 type Description: record {
@@ -133,62 +133,62 @@ function parse(unparsed_version: string): Description
 		{
 		# The regular expression should match the complete version number
 		# and software name.
-		local version_parts = split_n(unparsed_version, /\/?( [\(])?v?[0-9\-\._, ]{2,}/, T, 1);
-		if ( 1 in version_parts )
+		local version_parts = split_string_n(unparsed_version, /\/?( [\(])?v?[0-9\-\._, ]{2,}/, T, 1);
+		if ( 0 in version_parts )
 			{
-			if ( /^\(/ in version_parts[1] )
-				software_name = strip(sub(version_parts[1], /[\(]/, ""));
+			if ( /^\(/ in version_parts[0] )
+				software_name = strip(sub(version_parts[0], /[\(]/, ""));
 			else
-				software_name = strip(version_parts[1]);
+				software_name = strip(version_parts[0]);
 			}
 		if ( |version_parts| >= 2 )
 			{
 			# Remove the name/version separator if it's left at the beginning
 			# of the version number from the previous split_all.
-			local sv = strip(version_parts[2]);
+			local sv = strip(version_parts[1]);
 			if ( /^[\/\-\._v\(]/ in sv )
-				sv = strip(sub(version_parts[2], /^\(?[\/\-\._v\(]/, ""));
-			local version_numbers = split_n(sv, /[\-\._,\[\(\{ ]/, F, 3);
-			if ( 5 in version_numbers && version_numbers[5] != "" )
-				v$addl = strip(version_numbers[5]);
-			else if ( 3 in version_parts && version_parts[3] != "" &&
-			          version_parts[3] != ")" )
+				sv = strip(sub(version_parts[1], /^\(?[\/\-\._v\(]/, ""));
+			local version_numbers = split_string_n(sv, /[\-\._,\[\(\{ ]/, F, 3);
+			if ( 4 in version_numbers && version_numbers[4] != "" )
+				v$addl = strip(version_numbers[4]);
+			else if ( 2 in version_parts && version_parts[2] != "" &&
+			          version_parts[2] != ")" )
 				{
-				if ( /^[[:blank:]]*\([a-zA-Z0-9\-\._[:blank:]]*\)/ in version_parts[3] )
+				if ( /^[[:blank:]]*\([a-zA-Z0-9\-\._[:blank:]]*\)/ in version_parts[2] )
 					{
-					v$addl = split_n(version_parts[3], /[\(\)]/, F, 2)[2];
+					v$addl = split_string_n(version_parts[2], /[\(\)]/, F, 2)[1];
 					}
 				else
 					{
-					local vp = split_n(version_parts[3], /[\-\._,;\[\]\(\)\{\} ]/, F, 3);
-					if ( |vp| >= 1 && vp[1] != "" )
+					local vp = split_string_n(version_parts[2], /[\-\._,;\[\]\(\)\{\} ]/, F, 3);
+					if ( |vp| >= 1 && vp[0] != "" )
+						{
+						v$addl = strip(vp[0]);
+						}
+					else if ( |vp| >= 2 && vp[1] != "" )
 						{
 						v$addl = strip(vp[1]);
 						}
-					else if ( |vp| >= 2 && vp[2] != "" )
+					else if ( |vp| >= 3 && vp[2] != "" )
 						{
 						v$addl = strip(vp[2]);
 						}
-					else if ( |vp| >= 3 && vp[3] != "" )
-						{
-						v$addl = strip(vp[3]);
-						}
 					else
 						{
-						v$addl = strip(version_parts[3]);
+						v$addl = strip(version_parts[2]);
 						}
 						
 					}
 				}
 			
-			if ( 4 in version_numbers && version_numbers[4] != "" )
-				v$minor3 = extract_count(version_numbers[4]);
 			if ( 3 in version_numbers && version_numbers[3] != "" )
-				v$minor2 = extract_count(version_numbers[3]);
+				v$minor3 = extract_count(version_numbers[3]);
 			if ( 2 in version_numbers && version_numbers[2] != "" )
-				v$minor = extract_count(version_numbers[2]);
+				v$minor2 = extract_count(version_numbers[2]);
 			if ( 1 in version_numbers && version_numbers[1] != "" )
-				v$major = extract_count(version_numbers[1]);
+				v$minor = extract_count(version_numbers[1]);
+			if ( 0 in version_numbers && version_numbers[0] != "" )
+				v$major = extract_count(version_numbers[0]);
 			}
 		}
 	
@@ -200,14 +200,14 @@ function parse_mozilla(unparsed_version: string): Description
 	{
 	local software_name = "<unknown browser>";
 	local v: Version;
-	local parts: table[count] of string;
+	local parts: string_vec;
 	
 	if ( /Opera [0-9\.]*$/ in unparsed_version )
 		{
 		software_name = "Opera";
-		parts = split_all(unparsed_version, /Opera [0-9\.]*$/);
-		if ( 2 in parts )
-			v = parse(parts[2])$version;
+		parts = split_string_all(unparsed_version, /Opera [0-9\.]*$/);
+		if ( 1 in parts )
+			v = parse(parts[1])$version;
 		}
 	else if ( / MSIE |Trident\// in unparsed_version )
 		{
@@ -222,28 +222,28 @@ function parse_mozilla(unparsed_version: string): Description
 			v = [$major=11,$minor=0];
 		else
 			{
-			parts = split_all(unparsed_version, /MSIE [0-9]{1,2}\.*[0-9]*b?[0-9]*/);
-			if ( 2 in parts )
-				v = parse(parts[2])$version;
+			parts = split_string_all(unparsed_version, /MSIE [0-9]{1,2}\.*[0-9]*b?[0-9]*/);
+			if ( 1 in parts )
+				v = parse(parts[1])$version;
 			}
 		}
 	else if ( /Version\/.*Safari\// in unparsed_version )
 		{
 		software_name = "Safari";
-		parts = split_all(unparsed_version, /Version\/[0-9\.]*/);
-		if ( 2 in parts )
+		parts = split_string_all(unparsed_version, /Version\/[0-9\.]*/);
+		if ( 1 in parts )
 			{
-			v = parse(parts[2])$version;
+			v = parse(parts[1])$version;
 			if ( / Mobile\/?.* Safari/ in unparsed_version )
 				v$addl = "Mobile";
 			}
 		}
 	else if ( /(Firefox|Netscape|Thunderbird)\/[0-9\.]*/ in unparsed_version )
 		{
-		parts = split_all(unparsed_version, /(Firefox|Netscape|Thunderbird)\/[0-9\.]*/);
-		if ( 2 in parts )
+		parts = split_string_all(unparsed_version, /(Firefox|Netscape|Thunderbird)\/[0-9\.]*/);
+		if ( 1 in parts )
 			{
-			local tmp_s = parse(parts[2]);
+			local tmp_s = parse(parts[1]);
 			software_name = tmp_s$name;
 			v = tmp_s$version;
 			}
@@ -251,48 +251,55 @@ function parse_mozilla(unparsed_version: string): Description
 	else if ( /Chrome\/.*Safari\// in unparsed_version )
 		{
 		software_name = "Chrome";
-		parts = split_all(unparsed_version, /Chrome\/[0-9\.]*/);
-		if ( 2 in parts )
-			v = parse(parts[2])$version;
+		parts = split_string_all(unparsed_version, /Chrome\/[0-9\.]*/);
+		if ( 1 in parts )
+			v = parse(parts[1])$version;
 		}
 	else if ( /^Opera\// in unparsed_version )
 		{
 		if ( /Opera M(ini|obi)\// in unparsed_version )
 			{
-			parts = split_all(unparsed_version, /Opera M(ini|obi)/);
-			if ( 2 in parts )
-				software_name = parts[2];
-			parts = split_all(unparsed_version, /Version\/[0-9\.]*/);
-			if ( 2 in parts )
-				v = parse(parts[2])$version;
+			parts = split_string_all(unparsed_version, /Opera M(ini|obi)/);
+			if ( 1 in parts )
+				software_name = parts[1];
+			parts = split_string_all(unparsed_version, /Version\/[0-9\.]*/);
+			if ( 1 in parts )
+				v = parse(parts[1])$version;
 			else
 				{
-				parts = split_all(unparsed_version, /Opera Mini\/[0-9\.]*/);
-				if ( 2 in parts )
-					v = parse(parts[2])$version;
+				parts = split_string_all(unparsed_version, /Opera Mini\/[0-9\.]*/);
+				if ( 1 in parts )
+					v = parse(parts[1])$version;
 				}
 			}
 		else
 			{
 			software_name = "Opera";
-			parts = split_all(unparsed_version, /Version\/[0-9\.]*/);
-			if ( 2 in parts )
-				v = parse(parts[2])$version;
+			parts = split_string_all(unparsed_version, /Version\/[0-9\.]*/);
+			if ( 1 in parts )
+				v = parse(parts[1])$version;
 			}
 		}
+	else if ( /AdobeAIR\/[0-9\.]*/ in unparsed_version )
+		{
+		software_name = "AdobeAIR";
+		parts = split_string_all(unparsed_version, /AdobeAIR\/[0-9\.]*/);
+		if ( 1 in parts )
+			v = parse(parts[1])$version;
+		}
 	else if ( /AppleWebKit\/[0-9\.]*/ in unparsed_version )
 		{
 		software_name = "Unspecified WebKit";
-		parts = split_all(unparsed_version, /AppleWebKit\/[0-9\.]*/);
-		if ( 2 in parts )
-			v = parse(parts[2])$version;
+		parts = split_string_all(unparsed_version, /AppleWebKit\/[0-9\.]*/);
+		if ( 1 in parts )
+			v = parse(parts[1])$version;
 		}
 	else if ( / Java\/[0-9]\./ in unparsed_version )
 		{
 		software_name = "Java";
-		parts = split_all(unparsed_version, /Java\/[0-9\._]*/);
-		if ( 2 in parts )
-			v = parse(parts[2])$version;
+		parts = split_string_all(unparsed_version, /Java\/[0-9\._]*/);
+		if ( 1 in parts )
+			v = parse(parts[1])$version;
 		}
 
 	return [$version=v, $unparsed_version=unparsed_version, $name=software_name];

scripts/base/frameworks/tunnels/main.bro

@@ -89,7 +89,7 @@ redef likely_server_ports += { ayiya_ports, teredo_ports, gtpv1_ports };
 
 event bro_init() &priority=5
 	{
-	Log::create_stream(Tunnel::LOG, [$columns=Info]);
+	Log::create_stream(Tunnel::LOG, [$columns=Info, $path="tunnel"]);
 
 	Analyzer::register_for_ports(Analyzer::ANALYZER_AYIYA, ayiya_ports);
 	Analyzer::register_for_ports(Analyzer::ANALYZER_TEREDO, teredo_ports);

scripts/base/init-bare.bro

@@ -333,8 +333,6 @@ type connection: record {
 	## to parse the same data. If so, all will be recorded. Also note that
 	## the recorded services are independent of any transport-level protocols.
 	service: set[string];
-	addl: string;	##< Deprecated.
-	hot: count;	##< Deprecated.
 	history: string;	##< State history of connections. See *history* in :bro:see:`Conn::Info`.
 	## A globally unique connection identifier. For each connection, Bro
 	## creates an ID that is very likely unique across independent Bro runs.
@@ -347,15 +345,22 @@ type connection: record {
 	## for the connection unless the :bro:id:`tunnel_changed` event is
 	## handled and reassigns this field to the new encapsulation.
 	tunnel: EncapsulatingConnVector &optional;
+
+	## The outer VLAN, if applicable, for this connection.
+	vlan: int &optional;
+
+	## The inner VLAN, if applicable, for this connection.
+	inner_vlan: int &optional;
 };
 
 ## Default amount of time a file can be inactive before the file analysis
 ## gives up and discards any internal state related to the file.
 const default_file_timeout_interval: interval = 2 mins &redef;
 
-## Default amount of bytes that file analysis will buffer before raising
-## :bro:see:`file_new`.
-const default_file_bof_buffer_size: count = 1024 &redef;
+## Default amount of bytes that file analysis will buffer in order to use
+## for mime type matching.  File analyzers attached at the time of mime type
+## matching or later, will receive a copy of this buffer.
+const default_file_bof_buffer_size: count = 4096 &redef;
 
 ## A file that Bro is analyzing.  This is Bro's type for describing the basic
 ## internal metadata collected about a "file", which is essentially just a
@@ -394,8 +399,10 @@ type fa_file: record {
 	## during the process of analysis e.g. due to dropped packets.
 	missing_bytes: count &default=0;
 
-	## The number of not all-in-sequence bytes in the file stream that
-	## were delivered to file analyzers due to reassembly buffer overflow.
+	## The number of bytes in the file stream that were not delivered to
+	## stream file analyzers.  Generally, this consists of bytes that
+	## couldn't be reassembled, either because reassembly simply isn't
+	## enabled, or due to size limitations of the reassembly buffer.
 	overflow_bytes: count &default=0;
 
 	## The amount of time between receiving new data for this file that
@@ -409,18 +416,16 @@ type fa_file: record {
 	## The content of the beginning of a file up to *bof_buffer_size* bytes.
 	## This is also the buffer that's used for file/mime type detection.
 	bof_buffer: string &optional;
-
-	## The mime type of the strongest file magic signature matches against
-	## the data chunk in *bof_buffer*, or in the cases where no buffering
-	## of the beginning of file occurs, an initial guess of the mime type
-	## based on the first data seen.
-	mime_type: string &optional;
-
-	## All mime types that matched file magic signatures against the data
-	## chunk in *bof_buffer*, in order of their strength value.
-	mime_types: mime_matches &optional;
 } &redef;
 
+## Metadata that's been inferred about a particular file.
+type fa_metadata: record {
+	## The strongest matching mime type if one was discovered.
+	mime_type: string &optional;
+	## All matching mime types if any were discovered.
+	mime_types: mime_matches &optional;
+};
+
 ## Fields of a SYN packet.
 ##
 ## .. bro:see:: connection_SYN_packet
@@ -447,6 +452,7 @@ type NetStats: record {
 	## packet capture system, this value may not be available and will then
 	## be always set to zero.
 	pkts_link:    count &default=0;
+	bytes_recvd:  count &default=0;	##< Bytes received by Bro.
 };
 
 ## Statistics about Bro's resource consumption.
@@ -740,6 +746,7 @@ type pcap_packet: record {
 	caplen: count;	##< The number of bytes captured (<= *len*).
 	len: count;	##< The length of the packet in bytes, including link-level header.
 	data: string;	##< The payload of the packet, including link-level header.
+	link_type: link_encap;	##< Layer 2 link encapsulation type.
 };
 
 ## GeoIP location information.
@@ -935,7 +942,7 @@ const tcp_storm_interarrival_thresh = 1 sec &redef;
 ## seeing our peer's ACKs.  Set to zero to turn off this determination.
 ##
 ## .. bro:see:: tcp_max_above_hole_without_any_acks tcp_excessive_data_without_further_acks
-const tcp_max_initial_window = 4096 &redef;
+const tcp_max_initial_window = 16384 &redef;
 
 ## If we're not seeing our peer's ACKs, the maximum volume of data above a
 ## sequence hole that we'll tolerate before assuming that there's been a packet
@@ -943,7 +950,7 @@ const tcp_max_initial_window = 4096 &redef;
 ## don't ever give up.
 ##
 ## .. bro:see:: tcp_max_initial_window tcp_excessive_data_without_further_acks
-const tcp_max_above_hole_without_any_acks = 4096 &redef;
+const tcp_max_above_hole_without_any_acks = 16384 &redef;
 
 ## If we've seen this much data without any of it being acked, we give up
 ## on that connection to avoid memory exhaustion due to buffering all that
@@ -954,6 +961,11 @@ const tcp_max_above_hole_without_any_acks = 4096 &redef;
 ## .. bro:see:: tcp_max_initial_window tcp_max_above_hole_without_any_acks
 const tcp_excessive_data_without_further_acks = 10 * 1024 * 1024 &redef;
 
+## Number of TCP segments to buffer beyond what's been acknowledged already
+## to detect retransmission inconsistencies. Zero disables any additonal
+## buffering.
+const tcp_max_old_segments = 0 &redef;
+
 ## For services without a handler, these sets define originator-side ports
 ## that still trigger reassembly.
 ##
@@ -1087,27 +1099,6 @@ const ENDIAN_LITTLE = 1;	##< Little endian.
 const ENDIAN_BIG = 2;	##< Big endian.
 const ENDIAN_CONFUSED = 3;	##< Tried to determine endian, but failed.
 
-## Deprecated.
-function append_addl(c: connection, addl: string)
-	{
-	if ( c$addl == "" )
-		c$addl= addl;
-
-	else if ( addl !in c$addl )
-		c$addl = fmt("%s %s", c$addl, addl);
-	}
-
-## Deprecated.
-function append_addl_marker(c: connection, addl: string, marker: string)
-	{
-	if ( c$addl == "" )
-		c$addl= addl;
-
-	else if ( addl !in c$addl )
-		c$addl = fmt("%s%s%s", c$addl, marker, addl);
-	}
-
-
 # Values for :bro:see:`set_contents_file` *direction* argument.
 # todo:: these should go into an enum to make them autodoc'able
 const CONTENTS_NONE = 0;	##< Turn off recording of contents.
@@ -1516,6 +1507,34 @@ type pkt_hdr: record {
 	icmp: icmp_hdr &optional;	##< The ICMP header if an ICMP packet.
 };
 
+## Values extracted from the layer 2 header.
+##
+## .. bro:see:: pkt_hdr
+type l2_hdr: record {
+	encap: link_encap;      ##< L2 link encapsulation.
+	len: count;		##< Total frame length on wire.
+	cap_len: count;		##< Captured length.
+	src: string &optional;	##< L2 source (if Ethernet).
+	dst: string &optional;	##< L2 destination (if Ethernet).
+	vlan: count &optional;	##< Outermost VLAN tag if any (and Ethernet).
+	inner_vlan: count &optional;	##< Innermost VLAN tag if any (and Ethernet).
+	eth_type: count &optional;	##< Innermost Ethertype (if Ethernet).
+	proto: layer3_proto;	##< L3 protocol.
+};
+
+## A raw packet header, consisting of L2 header and everything in
+## :bro:id:`pkt_hdr`. .
+##
+## .. bro:see:: raw_packet pkt_hdr
+type raw_pkt_hdr: record {
+	l2: l2_hdr;			##< The layer 2 header.
+	ip: ip4_hdr &optional;		##< The IPv4 header if an IPv4 packet.
+	ip6: ip6_hdr &optional;		##< The IPv6 header if an IPv6 packet.
+	tcp: tcp_hdr &optional;		##< The TCP header if a TCP packet.
+	udp: udp_hdr &optional;		##< The UDP header if a UDP packet.
+	icmp: icmp_hdr &optional;	##< The ICMP header if an ICMP packet.
+};
+
 ## A Teredo origin indication header.  See :rfc:`4380` for more information
 ## about the Teredo protocol.
 ##
@@ -2222,6 +2241,41 @@ export {
 	const heartbeat_interval = 1.0 secs &redef;
 }
 
+module SSH;
+
+export {
+	## The client and server each have some preferences for the algorithms used
+	## in each direction.
+	type Algorithm_Prefs: record {
+		## The algorithm preferences for client to server communication
+		client_to_server: vector of string &optional;
+		## The algorithm preferences for server to client communication
+		server_to_client: vector of string &optional;
+	};
+
+	## This record lists the preferences of an SSH endpoint for
+	## algorithm selection. During the initial :abbr:`SSH (Secure Shell)`
+	## key exchange, each endpoint lists the algorithms
+	## that it supports, in order of preference. See
+	## :rfc:`4253#section-7.1` for details.
+	type Capabilities: record {
+		## Key exchange algorithms
+		kex_algorithms:             string_vec;
+		## The algorithms supported for the server host key
+		server_host_key_algorithms: string_vec;
+		## Symmetric encryption algorithm preferences
+		encryption_algorithms:      Algorithm_Prefs;
+		## Symmetric MAC algorithm preferences
+		mac_algorithms:             Algorithm_Prefs;
+		## Compression algorithm preferences
+		compression_algorithms:     Algorithm_Prefs;
+		## Language preferences
+		languages:                  Algorithm_Prefs &optional;
+		## Are these the capabilities of the server?
+		is_server:                  bool;
+	};
+}
+
 module GLOBAL;
 
 ## An NTP message.
@@ -2852,7 +2906,7 @@ global dns_skip_all_addl = T &redef;
 
 ## If a DNS request includes more than this many queries, assume it's non-DNS
 ## traffic and do not process it.  Set to 0 to turn off this functionality.
-global dns_max_queries = 5;
+global dns_max_queries = 25 &redef;
 
 ## HTTP session statistics.
 ##
@@ -2915,6 +2969,145 @@ type irc_join_info: record {
 ## .. bro:see:: irc_join_message
 type irc_join_list: set[irc_join_info];
 
+module PE;
+export {
+type PE::DOSHeader: record {
+	## The magic number of a portable executable file ("MZ").
+	signature                : string;
+	## The number of bytes in the last page that are used.
+	used_bytes_in_last_page  : count;
+	## The number of pages in the file that are part of the PE file itself.
+	file_in_pages            : count;
+	## Number of relocation entries stored after the header.
+	num_reloc_items          : count;
+	## Number of paragraphs in the header.
+	header_in_paragraphs     : count;
+	## Number of paragraps of additional memory that the program will need.
+	min_extra_paragraphs     : count;
+	## Maximum number of paragraphs of additional memory.
+	max_extra_paragraphs     : count;
+	## Relative value of the stack segment.
+	init_relative_ss         : count;
+	## Initial value of the SP register.
+	init_sp                  : count;
+	## Checksum. The 16-bit sum of all words in the file should be 0. Normally not set.
+	checksum                 : count;
+	## Initial value of the IP register.
+	init_ip                  : count;
+	## Initial value of the CS register (relative to the initial segment).
+	init_relative_cs         : count;
+	## Offset of the first relocation table.
+	addr_of_reloc_table      : count;
+	## Overlays allow you to append data to the end of the file. If this is the main program,
+	## this will be 0.
+	overlay_num              : count;
+	## OEM identifier.
+	oem_id                   : count;
+	## Additional OEM info, specific to oem_id.
+	oem_info                 : count;
+	## Address of the new EXE header.
+	addr_of_new_exe_header   : count;
+};
+
+type PE::FileHeader: record {
+	## The target machine that the file was compiled for.
+	machine              : count;
+	## The time that the file was created at.
+	ts                   : time;
+	## Pointer to the symbol table.
+	sym_table_ptr        : count;
+	## Number of symbols.
+	num_syms             : count;
+	## The size of the optional header.
+	optional_header_size : count;
+	## Bit flags that determine if this file is executable, non-relocatable, and/or a DLL.
+	characteristics      : set[count];
+};
+
+type PE::OptionalHeader: record {
+	## PE32 or PE32+ indicator.
+	magic                   : count;
+	## The major version of the linker used to create the PE.
+	major_linker_version    : count;
+	## The minor version of the linker used to create the PE.
+	minor_linker_version    : count;
+	## Size of the .text section.
+	size_of_code            : count;
+	## Size of the .data section.
+	size_of_init_data       : count;
+	## Size of the .bss section.
+	size_of_uninit_data     : count;
+	## The relative virtual address (RVA) of the entry point.
+	addr_of_entry_point     : count;
+	## The relative virtual address (RVA) of the .text section.
+	base_of_code            : count;
+	## The relative virtual address (RVA) of the .data section.
+	base_of_data            : count &optional;
+	## Preferred memory location for the image to be based at.
+	image_base              : count;
+	## The alignment (in bytes) of sections when they're loaded in memory.
+	section_alignment       : count;
+	## The alignment (in bytes) of the raw data of sections.
+	file_alignment          : count;
+	## The major version of the required OS.
+	os_version_major        : count;
+	## The minor version of the required OS.
+	os_version_minor        : count;
+	## The major version of this image.
+	major_image_version     : count;
+	## The minor version of this image.
+	minor_image_version     : count;
+	## The major version of the subsystem required to run this file.
+	major_subsys_version    : count;
+	## The minor version of the subsystem required to run this file.
+	minor_subsys_version    : count;
+	## The size (in bytes) of the iamge as the image is loaded in memory.
+	size_of_image           : count;
+	## The size (in bytes) of the headers, rounded up to file_alignment.
+	size_of_headers         : count;
+	## The image file checksum.
+	checksum                : count;
+	## The subsystem that's required to run this image.
+	subsystem               : count;
+	## Bit flags that determine how to execute or load this file.
+	dll_characteristics     : set[count];
+	## A vector with the sizes of various tables and strings that are
+	## defined in the optional header data directories. Examples include
+	## the import table, the resource table, and debug information.
+	table_sizes             : vector of count;
+
+};
+
+## Record for Portable Executable (PE) section headers.
+type PE::SectionHeader: record {
+	## The name of the section
+	name             : string;
+	## The total size of the section when loaded into memory.
+	virtual_size     : count;
+	## The relative virtual address (RVA) of the section.
+	virtual_addr     : count;
+	## The size of the initialized data for the section, as it is
+	## in the file on disk.
+	size_of_raw_data : count;
+	## The virtual address of the initialized dat for the section,
+	## as it is in the file on disk.
+	ptr_to_raw_data  : count;
+	## The file pointer to the beginning of relocation entries for
+	## the section.
+	ptr_to_relocs    : count;
+	## The file pointer to the beginning of line-number entries for
+	## the section.
+	ptr_to_line_nums : count;
+	## The number of relocation entries for the section.
+	num_of_relocs    : count;
+	## The number of line-number entrie for the section.
+	num_of_line_nums : count;
+	## Bit-flags that describe the characteristics of the section.
+	characteristics  : set[count];
+};
+}
+module GLOBAL;
+
 ## Deprecated.
 ##
 ## .. todo:: Remove. It's still declared internally but doesn't seem  used anywhere
@@ -3039,60 +3232,6 @@ global generate_OS_version_event: set[subnet] &redef;
 # number>``), which were seen during the sample.
 type load_sample_info: set[string];
 
-## ID for NetFlow header. This is primarily a means to sort together NetFlow
-## headers and flow records at the script level.
-type nfheader_id: record {
-	## Name of the NetFlow file (e.g., ``netflow.dat``) or the receiving
-	## socket address (e.g., ``127.0.0.1:5555``), or an explicit name if
-	## specified to ``-y`` or ``-Y``.
-	rcvr_id: string;
-	## A serial number, ignoring any overflows.
-	pdu_id: count;
-};
-
-## A NetFlow v5 header.
-##
-## .. bro:see:: netflow_v5_header
-type nf_v5_header: record {
-	h_id: nfheader_id;	##< ID for sorting.
-	cnt: count;	##< TODO.
-	sysuptime: interval;	##< Router's uptime.
-	exporttime: time;	##< When the data was exported.
-	flow_seq: count;	##< Sequence number.
-	eng_type: count;	##< Engine type.
-	eng_id: count;	##< Engine ID.
-	sample_int: count;	##< Sampling interval.
-	exporter: addr;	##< Exporter address.
-};
-
-## A NetFlow v5 record.
-##
-## .. bro:see:: netflow_v5_record
-type nf_v5_record: record {
-	h_id: nfheader_id;	##< ID for sorting.
-	id: conn_id;	##< Connection ID.
-	nexthop: addr;	##< Address of next hop.
-	input: count;	##< Input interface.
-	output: count;	##< Output interface.
-	pkts: count;	##< Number of packets.
-	octets: count;	##< Number of bytes.
-	first: time;	##< Timestamp of first packet.
-	last: time;	##< Timestamp of last packet.
-	tcpflag_fin: bool;	##< FIN flag for TCP flows.
-	tcpflag_syn: bool;	##< SYN flag for TCP flows.
-	tcpflag_rst: bool;	##< RST flag for TCP flows.
-	tcpflag_psh: bool;	##< PSH flag for TCP flows.
-	tcpflag_ack: bool;	##< ACK flag for TCP flows.
-	tcpflag_urg: bool;	##< URG flag for TCP flows.
-	proto: count;	##< IP protocol.
-	tos: count;	##< Type of service.
-	src_as: count;	##< Source AS.
-	dst_as: count;	##< Destination AS.
-	src_mask: count;	##< Source mask.
-	dst_mask: count;	##< Destination mask.
-};
-
-
 ## A BitTorrent peer.
 ##
 ## .. bro:see:: bittorrent_peer_set
@@ -3178,19 +3317,20 @@ export {
 module X509;
 export {
 	type Certificate: record {
-		version: count;	##< Version number.
-		serial: string;	##< Serial number.
-		subject: string;	##< Subject.
-		issuer: string;	##< Issuer.
-		not_valid_before: time;	##< Timestamp before when certificate is not valid.
-		not_valid_after: time;	##< Timestamp after when certificate is not valid.
-		key_alg: string;	##< Name of the key algorithm
-		sig_alg: string;	##< Name of the signature algorithm
-		key_type: string &optional;	##< Key type, if key parseable by openssl (either rsa, dsa or ec)
-		key_length: count &optional;	##< Key length in bits
-		exponent: string &optional;	##< Exponent, if RSA-certificate
-		curve: string &optional;	##< Curve, if EC-certificate
-	} &log;
+		version: count &log;	##< Version number.
+		serial: string &log;	##< Serial number.
+		subject: string &log;	##< Subject.
+		issuer: string &log;	##< Issuer.
+		cn: string &optional; ##< Last (most specific) common name.
+		not_valid_before: time &log;	##< Timestamp before when certificate is not valid.
+		not_valid_after: time &log;	##< Timestamp after when certificate is not valid.
+		key_alg: string &log;	##< Name of the key algorithm
+		sig_alg: string &log;	##< Name of the signature algorithm
+		key_type: string &optional &log;	##< Key type, if key parseable by openssl (either rsa, dsa or ec)
+		key_length: count &optional &log;	##< Key length in bits
+		exponent: string &optional &log;	##< Exponent, if RSA-certificate
+		curve: string &optional &log;	##< Curve, if EC-certificate
+	};
 
 	type Extension: record {
 		name: string;	##< Long name of extension. oid if name not known
@@ -3251,7 +3391,44 @@ export {
 		attributes    : RADIUS::Attributes &optional;
 	};
 }
-module GLOBAL;
+
+module RDP;
+export {
+	type RDP::EarlyCapabilityFlags: record {
+		support_err_info_pdu:       bool;
+		want_32bpp_session:         bool;
+		support_statusinfo_pdu:     bool;
+		strong_asymmetric_keys:     bool;
+		support_monitor_layout_pdu: bool;
+		support_netchar_autodetect: bool;
+		support_dynvc_gfx_protocol: bool;
+		support_dynamic_time_zone:  bool;
+		support_heartbeat_pdu:      bool;
+	};
+
+	type RDP::ClientCoreData: record {
+		version_major:          count;
+		version_minor:          count;
+		desktop_width:          count;
+		desktop_height:         count;
+		color_depth:            count;
+		sas_sequence:           count;
+		keyboard_layout:        count;
+		client_build:           count;
+		client_name:            string;
+		keyboard_type:          count;
+		keyboard_sub:           count;
+		keyboard_function_key:  count;
+		ime_file_name:          string;
+		post_beta2_color_depth: count  &optional;
+		client_product_id:      string &optional;
+		serial_number:          count  &optional;
+		high_color_depth:       count  &optional;
+		supported_color_depths: count  &optional;
+		ec_flags:               RDP::EarlyCapabilityFlags &optional;
+		dig_product_id:         string &optional;
+	};
+}
 
 @load base/bif/plugins/Bro_SNMP.types.bif
 
@@ -3375,6 +3552,186 @@ export {
 	};
 }
 
+@load base/bif/plugins/Bro_KRB.types.bif
+
+module KRB;
+export {
+	## KDC Options. See :rfc:`4120`
+	type KRB::KDC_Options: record {
+		## The ticket to be issued should have its forwardable flag set.
+		forwardable		: bool;
+		## A (TGT) request for forwarding.
+		forwarded		: bool;
+		## The ticket to be issued should have its proxiable flag set.
+		proxiable		: bool;
+		## A request for a proxy.
+		proxy			: bool;
+		## The ticket to be issued should have its may-postdate flag set.
+		allow_postdate		: bool;
+		## A request for a postdated ticket.
+		postdated		: bool;
+		## The ticket to be issued should have its renewable  flag set.
+		renewable		: bool;
+		## Reserved for opt_hardware_auth
+		opt_hardware_auth	: bool;
+		## Request that the KDC not check the transited field of a TGT against
+		## the policy of the local realm before it will issue derivative tickets
+		## based on the TGT.
+		disable_transited_check	: bool;
+		## If a ticket with the requested lifetime cannot be issued, a renewable
+		## ticket is acceptable
+		renewable_ok		: bool;
+		## The ticket for the end server is to be encrypted in the session key
+		## from the additional TGT provided
+		enc_tkt_in_skey		: bool;
+		## The request is for a renewal
+		renew			: bool;
+		## The request is to validate a postdated ticket.
+		validate		: bool;
+	};
+
+	## AP Options. See :rfc:`4120`
+	type KRB::AP_Options: record {
+		## Indicates that user-to-user-authentication is in use
+		use_session_key	: bool;
+		## Mutual authentication is required
+		mutual_required	: bool;
+	};
+
+	## Used in a few places in the Kerberos analyzer for elements
+	## that have a type and a string value.
+	type KRB::Type_Value: record {
+		## The data type
+		data_type	: count;
+		## The data value
+		val 		: string;
+	};
+
+	type KRB::Type_Value_Vector: vector of KRB::Type_Value;
+
+	## A Kerberos host address See :rfc:`4120`.
+	type KRB::Host_Address: record {
+		## IPv4 or IPv6 address
+		ip	: addr &log &optional;
+		## NetBIOS address
+		netbios : string &log &optional;
+		## Some other type that we don't support yet
+		unknown : KRB::Type_Value &optional;
+	};
+
+	type KRB::Host_Address_Vector: vector of KRB::Host_Address;
+
+	## The data from the SAFE message. See :rfc:`4120`.
+	type KRB::SAFE_Msg: record {
+		## Protocol version number (5 for KRB5)
+		pvno		: count;
+		## The message type (20 for SAFE_MSG)
+		msg_type	: count;
+		## The application-specific data that is being passed
+		## from the sender to the reciever
+		data		: string;
+		## Current time from the sender of the message
+		timestamp	: time &optional;
+		## Sequence number used to detect replays
+		seq		: count &optional;
+		## Sender address
+		sender		: Host_Address &optional;
+		## Recipient address
+		recipient    	: Host_Address &optional;
+	};
+
+	## The data from the ERROR_MSG message. See :rfc:`4120`.
+	type KRB::Error_Msg: record {
+		## Protocol version number (5 for KRB5)
+		pvno		: count;
+		## The message type (30 for ERROR_MSG)
+		msg_type	: count;
+		## Current time on the client
+		client_time	: time &optional;
+		## Current time on the server
+		server_time	: time;
+		## The specific error code
+		error_code	: count;
+		## Realm of the ticket
+		client_realm	: string &optional;
+		## Name on the ticket
+		client_name	: string &optional;
+		## Realm of the service
+		service_realm	: string;
+		## Name of the service
+		service_name	: string;
+		## Additional text to explain the error
+		error_text	: string &optional;
+		## Optional pre-authentication data
+		pa_data		: vector of KRB::Type_Value &optional;
+	};
+
+	## A Kerberos ticket. See :rfc:`4120`.
+	type KRB::Ticket: record {
+		## Protocol version number (5 for KRB5)
+		pvno		: count;
+		## Realm
+		realm		: string;
+		## Name of the service
+		service_name	: string;
+		## Cipher the ticket was encrypted with
+		cipher		: count;
+	};
+
+	type KRB::Ticket_Vector: vector of KRB::Ticket;
+
+	## The data from the AS_REQ and TGS_REQ messages. See :rfc:`4120`.
+	type KRB::KDC_Request: record {
+		## Protocol version number (5 for KRB5)
+		pvno			: count;
+		## The message type (10 for AS_REQ, 12 for TGS_REQ)
+		msg_type		: count;
+		## Optional pre-authentication data
+		pa_data			: vector of KRB::Type_Value &optional;
+		## Options specified in the request
+		kdc_options		: KRB::KDC_Options;
+		## Name on the ticket
+		client_name		: string &optional;
+
+		## Realm of the service
+		service_realm		: string;
+		## Name of the service
+		service_name		: string &optional;
+		## Time the ticket is good from
+		from			: time &optional;
+		## Time the ticket is good till
+		till			: time;
+		## The requested renew-till time
+		rtime			: time &optional;
+
+		## A random nonce generated by the client
+		nonce			: count;
+		## The desired encryption algorithms, in order of preference
+		encryption_types	: vector of count;
+		## Any additional addresses the ticket should be valid for
+		host_addrs		: vector of KRB::Host_Address &optional;
+		## Additional tickets may be included for certain transactions
+		additional_tickets	: vector of KRB::Ticket &optional;
+	};
+
+	## The data from the AS_REQ and TGS_REQ messages. See :rfc:`4120`.
+	type KRB::KDC_Response: record {
+		## Protocol version number (5 for KRB5)
+		pvno			: count;
+		## The message type (11 for AS_REP, 13 for TGS_REP)
+		msg_type		: count;
+		## Optional pre-authentication data
+		pa_data			: vector of KRB::Type_Value &optional;
+		## Realm on the ticket
+		client_realm		: string &optional;
+		## Name on the service
+		client_name		: string;
+
+		## The ticket that was issued
+		ticket			: KRB::Ticket;
+	};
+}
+
 module GLOBAL;
 
 @load base/bif/event.bif
@@ -3537,6 +3894,11 @@ const forward_remote_events = F &redef;
 ##    more sophisticated script-level communication framework.
 const forward_remote_state_changes = F &redef;
 
+## The number of IO chunks allowed to be buffered between the child
+## and parent process of remote communication before Bro starts dropping
+## connections to remote peers in an attempt to catch up.
+const chunked_io_buffer_soft_cap = 800000 &redef;
+
 ## Place-holder constant indicating "no peer".
 const PEER_ID_NONE = 0;
 
@@ -3697,20 +4059,11 @@ export {
 	## Toggle whether to do GRE decapsulation.
 	const enable_gre = T &redef;
 
-	## With this option set, the Teredo analysis will first check to see if
-	## other protocol analyzers have confirmed that they think they're
-	## parsing the right protocol and only continue with Teredo tunnel
-	## decapsulation if nothing else has yet confirmed.  This can help
-	## reduce false positives of UDP traffic (e.g. DNS) that also happens
-	## to have a valid Teredo encapsulation.
-	const yielding_teredo_decapsulation = T &redef;
-
 	## With this set, the Teredo analyzer waits until it sees both sides
 	## of a connection using a valid Teredo encapsulation before issuing
 	## a :bro:see:`protocol_confirmation`.  If it's false, the first
 	## occurrence of a packet with valid Teredo encapsulation causes a
-	## confirmation.  Both cases are still subject to effects of
-	## :bro:see:`Tunnel::yielding_teredo_decapsulation`.
+	## confirmation.
 	const delay_teredo_confirmation = T &redef;
 
 	## With this set, the GTP analyzer waits until the most-recent upflow
@@ -3726,7 +4079,6 @@ export {
 	## (includes GRE tunnels).
 	const ip_tunnel_timeout = 24hrs &redef;
 } # end export
-module GLOBAL;
 
 module Reporter;
 export {
@@ -3745,10 +4097,18 @@ export {
 	## external harness and shouldn't output anything to the console.
 	const errors_to_stderr = T &redef;
 }
-module GLOBAL;
 
-## Number of bytes per packet to capture from live interfaces.
-const snaplen = 8192 &redef;
+module Pcap;
+export {
+	## Number of bytes per packet to capture from live interfaces.
+	const snaplen = 8192 &redef;
+
+	## Number of Mbytes to provide as buffer space when capturing from live
+	## interfaces.
+	const bufsize = 128 &redef;
+} # end export
+
+module GLOBAL;
 
 ## Seed for hashes computed internally for probabilistic data structures. Using
 ## the same value here will make the hashes compatible between independent Bro
@@ -3762,6 +4122,7 @@ const bits_per_uid: count = 96 &redef;
 
 # Load these frameworks here because they use fairly deep integration with
 # BiFs and script-land defined types.
+@load base/frameworks/broker
 @load base/frameworks/logging
 @load base/frameworks/input
 @load base/frameworks/analyzer

scripts/base/init-default.bro

@@ -45,9 +45,13 @@
 @load base/protocols/ftp
 @load base/protocols/http
 @load base/protocols/irc
+@load base/protocols/krb
 @load base/protocols/modbus
+@load base/protocols/mysql
 @load base/protocols/pop3
 @load base/protocols/radius
+@load base/protocols/rdp
+@load base/protocols/sip
 @load base/protocols/smb
 @load base/protocols/snmp
 @load base/protocols/smtp
@@ -57,6 +61,7 @@
 @load base/protocols/syslog
 @load base/protocols/tunnels
 
+@load base/files/pe
 @load base/files/hash
 @load base/files/extract
 @load base/files/unified2

scripts/base/misc/find-checksum-offloading.bro

@@ -50,7 +50,7 @@ event ChecksumOffloading::check()
 			bad_checksum_msg += "UDP";
 			}
 
-		local message = fmt("Your %s invalid %s checksums, most likely from NIC checksum offloading.", packet_src, bad_checksum_msg);
+		local message = fmt("Your %s invalid %s checksums, most likely from NIC checksum offloading.  By default, packets with invalid checksums are discarded by Bro unless using the -C command-line option or toggling the 'ignore_checksums' variable.  Alternatively, disable checksum offloading by the network adapter to ensure Bro analyzes the actual checksums that are transmitted.", packet_src, bad_checksum_msg);
 		Reporter::warning(message);
 		done = T;
 		}

scripts/base/protocols/conn/__load__.bro

@@ -2,3 +2,4 @@
 @load ./contents
 @load ./inactivity
 @load ./polling
+@load ./thresholds

scripts/base/protocols/conn/main.bro

@@ -62,6 +62,12 @@ export {
 		## field will be left empty at all times.
 		local_orig:   bool            &log &optional;
 
+		## If the connection is responded to locally, this value will be T.
+		## If it was responded to remotely it will be F.  In the case that
+		## the :bro:id:`Site::local_nets` variable is undefined, this
+		## field will be left empty at all times.
+		local_resp:   bool            &log &optional;
+
 		## Indicates the number of bytes missed in content gaps, which
 		## is representative of packet loss.  A value other than zero
 		## will normally cause protocol analysis to fail but some
@@ -81,7 +87,8 @@ export {
 		## f       packet with FIN bit set
 		## r       packet with RST bit set
 		## c       packet with a bad checksum
-		## i       inconsistent packet (e.g. SYN+RST bits both set)
+		## i       inconsistent packet (e.g. FIN+RST bits set)
+		## q       multi-flag packet (SYN+FIN or SYN+RST bits set)
 		## ======  ====================================================
 		##
 		## If the event comes from the originator, the letter is in
@@ -121,7 +128,7 @@ redef record connection += {
 
 event bro_init() &priority=5
 	{
-	Log::create_stream(Conn::LOG, [$columns=Info, $ev=log_conn]);
+	Log::create_stream(Conn::LOG, [$columns=Info, $ev=log_conn, $path="conn"]);
 	}
 
 function conn_state(c: connection, trans: transport_proto): string
@@ -201,7 +208,10 @@ function set_conn(c: connection, eoc: bool)
 		add c$conn$tunnel_parents[c$tunnel[|c$tunnel|-1]$uid];
 	c$conn$proto=get_port_transport_proto(c$id$resp_p);
 	if( |Site::local_nets| > 0 )
+		{
 		c$conn$local_orig=Site::is_local_addr(c$id$orig_h);
+		c$conn$local_resp=Site::is_local_addr(c$id$resp_h);
+		}
 
 	if ( eoc )
 		{

scripts/base/protocols/conn/thresholds.bro

@@ -0,0 +1,256 @@
+##! Implements a generic API to throw events when a connection crosses a
+##! fixed threshold of bytes or packets.
+
+module ConnThreshold;
+
+export {
+
+	type Thresholds: record {
+		orig_byte: set[count] &default=count_set(); ##< current originator byte thresholds we watch for
+		resp_byte: set[count] &default=count_set(); ##< current responder byte thresholds we watch for
+		orig_packet: set[count] &default=count_set(); ##< corrent originator packet thresholds we watch for
+		resp_packet: set[count] &default=count_set(); ##< corrent responder packet thresholds we watch for
+	};
+
+	## Sets a byte threshold for connection sizes, adding it to potentially already existing thresholds.
+	## conn_bytes_threshold_crossed will be raised for each set threshold.
+	##
+	## cid: The connection id.
+	##
+	## threshold: Threshold in bytes.
+	##
+	## is_orig: If true, threshold is set for bytes from originator, otherwise for bytes from responder.
+	##
+	## Returns: T on success, F on failure.
+	global set_bytes_threshold: function(c: connection, threshold: count, is_orig: bool): bool;
+
+	## Sets a packet threshold for connection sizes, adding it to potentially already existing thresholds.
+	## conn_packets_threshold_crossed will be raised for each set threshold.
+	##
+	## cid: The connection id.
+	##
+	## threshold: Threshold in packets.
+	##
+	## is_orig: If true, threshold is set for packets from originator, otherwise for packets from responder.
+	##
+	## Returns: T on success, F on failure.
+	global set_packets_threshold: function(c: connection, threshold: count, is_orig: bool): bool;
+
+	## Deletes a byte threshold for connection sizes.
+	##
+	## cid: The connection id.
+	##
+	## threshold: Threshold in bytes to remove.
+	##
+	## is_orig: If true, threshold is removed for packets from originator, otherwhise for packets from responder.
+	##
+	## Returns: T on success, F on failure.
+	global delete_bytes_threshold: function(c: connection, threshold: count, is_orig: bool): bool;
+
+	## Deletes a packet threshold for connection sizes.
+	##
+	## cid: The connection id.
+	##
+	## threshold: Threshold in packets.
+	##
+	## is_orig: If true, threshold is removed for packets from originator, otherwise for packets from responder.
+	##
+	## Returns: T on success, F on failure.
+	global delete_packets_threshold: function(c: connection, threshold: count, is_orig: bool): bool;
+
+	## Generated for a connection that crossed a set byte threshold
+	##
+	## c: the connection
+	##
+	## threshold: the threshold that was set
+	##
+	## is_orig: True if the threshold was crossed by the originator of the connection
+	global bytes_threshold_crossed: event(c: connection, threshold: count, is_orig: bool);
+
+	## Generated for a connection that crossed a set byte threshold
+	##
+	## c: the connection
+	##
+	## threshold: the threshold that was set
+	##
+	## is_orig: True if the threshold was crossed by the originator of the connection
+	global packets_threshold_crossed: event(c: connection, threshold: count, is_orig: bool);
+}
+
+redef record connection += {
+	thresholds: ConnThreshold::Thresholds &optional;
+};
+
+function set_conn(c: connection)
+	{
+	if ( c?$thresholds )
+		return;
+
+	c$thresholds = Thresholds();
+	}
+
+function find_min_threshold(t: set[count]): count
+	{
+	if ( |t| == 0 )
+		return 0;
+
+	local first = T;
+	local min: count = 0;
+
+	for ( i in t )
+		{
+		if ( first )
+			{
+			min = i;
+			first = F;
+			}
+		else
+			{
+			if ( i < min )
+				min = i;
+			}
+		}
+
+	return min;
+	}
+
+function set_current_threshold(c: connection, bytes: bool, is_orig: bool): bool
+	{
+	local t: count = 0;
+	local cur: count = 0;
+
+	if ( bytes && is_orig )
+		{
+		t = find_min_threshold(c$thresholds$orig_byte);
+		cur = get_current_conn_bytes_threshold(c$id, is_orig);
+		}
+	else if ( bytes && ! is_orig )
+		{
+		t = find_min_threshold(c$thresholds$resp_byte);
+		cur = get_current_conn_bytes_threshold(c$id, is_orig);
+		}
+	else if ( ! bytes && is_orig )
+		{
+		t = find_min_threshold(c$thresholds$orig_packet);
+		cur = get_current_conn_packets_threshold(c$id, is_orig);
+		}
+	else if ( ! bytes && ! is_orig )
+		{
+		t = find_min_threshold(c$thresholds$resp_packet);
+		cur = get_current_conn_packets_threshold(c$id, is_orig);
+		}
+
+	if ( t == cur )
+		return T;
+
+	if ( bytes && is_orig )
+		return set_current_conn_bytes_threshold(c$id, t, T);
+	else if ( bytes && ! is_orig )
+		return set_current_conn_bytes_threshold(c$id, t, F);
+	else if ( ! bytes && is_orig )
+		return set_current_conn_packets_threshold(c$id, t, T);
+	else if ( ! bytes && ! is_orig )
+		return set_current_conn_packets_threshold(c$id, t, F);
+	}
+
+function set_bytes_threshold(c: connection, threshold: count, is_orig: bool): bool
+	{
+	set_conn(c);
+
+	if ( threshold == 0 )
+		return F;
+
+	if ( is_orig )
+		add c$thresholds$orig_byte[threshold];
+	else
+		add c$thresholds$resp_byte[threshold];
+
+	return set_current_threshold(c, T, is_orig);
+	}
+
+function set_packets_threshold(c: connection, threshold: count, is_orig: bool): bool
+	{
+	set_conn(c);
+
+	if ( threshold == 0 )
+		return F;
+
+	if ( is_orig )
+		add c$thresholds$orig_packet[threshold];
+	else
+		add c$thresholds$resp_packet[threshold];
+
+	return set_current_threshold(c, F, is_orig);
+	}
+
+function delete_bytes_threshold(c: connection, threshold: count, is_orig: bool): bool
+	{
+	set_conn(c);
+
+	if ( is_orig && threshold in c$thresholds$orig_byte )
+		{
+		delete c$thresholds$orig_byte[threshold];
+		set_current_threshold(c, T, is_orig);
+		return T;
+		}
+	else if ( ! is_orig && threshold in c$thresholds$resp_byte )
+		{
+		delete c$thresholds$resp_byte[threshold];
+		set_current_threshold(c, T, is_orig);
+		return T;
+		}
+
+	return F;
+	}
+
+function delete_packets_threshold(c: connection, threshold: count, is_orig: bool): bool
+	{
+	set_conn(c);
+
+	if ( is_orig && threshold in c$thresholds$orig_packet )
+		{
+		delete c$thresholds$orig_packet[threshold];
+		set_current_threshold(c, F, is_orig);
+		return T;
+		}
+	else if ( ! is_orig && threshold in c$thresholds$resp_packet )
+		{
+		delete c$thresholds$resp_packet[threshold];
+		set_current_threshold(c, F, is_orig);
+		return T;
+		}
+
+	return F;
+	}
+
+event conn_bytes_threshold_crossed(c: connection, threshold: count, is_orig: bool) &priority=5
+	{
+	if ( is_orig && threshold in c$thresholds$orig_byte )
+		{
+		delete c$thresholds$orig_byte[threshold];
+		event ConnThreshold::bytes_threshold_crossed(c, threshold, is_orig);
+		}
+	else if ( ! is_orig && threshold in c$thresholds$resp_byte )
+		{
+		delete c$thresholds$resp_byte[threshold];
+		event ConnThreshold::bytes_threshold_crossed(c, threshold, is_orig);
+		}
+
+	set_current_threshold(c, T, is_orig);
+	}
+
+event conn_packets_threshold_crossed(c: connection, threshold: count, is_orig: bool) &priority=5
+	{
+	if ( is_orig && threshold in c$thresholds$orig_packet )
+		{
+		delete c$thresholds$orig_packet[threshold];
+		event ConnThreshold::packets_threshold_crossed(c, threshold, is_orig);
+		}
+	else if ( ! is_orig && threshold in c$thresholds$resp_packet )
+		{
+		delete c$thresholds$resp_packet[threshold];
+		event ConnThreshold::packets_threshold_crossed(c, threshold, is_orig);
+		}
+
+	set_current_threshold(c, F, is_orig);
+	}

scripts/base/protocols/dhcp/main.bro

@@ -49,7 +49,7 @@ redef likely_server_ports += { 67/udp };
 
 event bro_init() &priority=5
 	{
-	Log::create_stream(DHCP::LOG, [$columns=Info, $ev=log_dhcp]);
+	Log::create_stream(DHCP::LOG, [$columns=Info, $ev=log_dhcp, $path="dhcp"]);
 	Analyzer::register_for_ports(Analyzer::ANALYZER_DHCP, ports);
 	}
 

scripts/base/protocols/dhcp/utils.bro

@@ -13,7 +13,7 @@ export {
 
 function reverse_ip(ip: addr): addr
 	{
-	local octets = split(cat(ip), /\./);
-	return to_addr(cat(octets[4], ".", octets[3], ".", octets[2], ".", octets[1]));
+	local octets = split_string(cat(ip), /\./);
+	return to_addr(cat(octets[3], ".", octets[2], ".", octets[1], ".", octets[0]));
 	}
 

scripts/base/protocols/dnp3/dpd.sig

@@ -5,5 +5,11 @@ signature dpd_dnp3_server {
 	ip-proto == tcp
 	payload /\x05\x64/
 	tcp-state responder
- 	enable "dnp3"
+ 	enable "dnp3_tcp"
+}
+
+signature dpd_dnp3_server_udp {
+	ip-proto == udp
+	payload /\x05\x64/
+	enable "dnp3_udp"
 }

scripts/base/protocols/dnp3/main.bro

@@ -31,16 +31,16 @@ redef record connection += {
 	dnp3: Info &optional;
 };
 
-const ports = { 20000/tcp };
+const ports = { 20000/tcp , 20000/udp };
 redef likely_server_ports += { ports };
 
 event bro_init() &priority=5
 	{
-	Log::create_stream(DNP3::LOG, [$columns=Info, $ev=log_dnp3]);
-	Analyzer::register_for_ports(Analyzer::ANALYZER_DNP3, ports);
+	Log::create_stream(DNP3::LOG, [$columns=Info, $ev=log_dnp3, $path="dnp3"]);
+	Analyzer::register_for_ports(Analyzer::ANALYZER_DNP3_TCP, ports);
 	}
 
-event dnp3_application_request_header(c: connection, is_orig: bool, fc: count)
+event dnp3_application_request_header(c: connection, is_orig: bool, application_control: count, fc: count)
 	{
 	if ( ! c?$dnp3 )
 		c$dnp3 = [$ts=network_time(), $uid=c$uid, $id=c$id];
@@ -49,7 +49,7 @@ event dnp3_application_request_header(c: connection, is_orig: bool, fc: count)
 	c$dnp3$fc_request = function_codes[fc];
 	}
 
-event dnp3_application_response_header(c: connection, is_orig: bool, fc: count, iin: count)
+event dnp3_application_response_header(c: connection, is_orig: bool, application_control: count, fc: count, iin: count)
 	{
 	if ( ! c?$dnp3 )
 		c$dnp3 = [$ts=network_time(), $uid=c$uid, $id=c$id];

scripts/base/protocols/dns/main.bro

@@ -150,7 +150,7 @@ redef likely_server_ports += { ports };
 
 event bro_init() &priority=5
 	{
-	Log::create_stream(DNS::LOG, [$columns=Info, $ev=log_dns]);
+	Log::create_stream(DNS::LOG, [$columns=Info, $ev=log_dns, $path="dns"]);
 	Analyzer::register_for_ports(Analyzer::ANALYZER_DNS, ports);
 	}
 
@@ -305,6 +305,9 @@ hook DNS::do_reply(c: connection, msg: dns_msg, ans: dns_answer, reply: string)
 
 	if ( ans$answer_type == DNS_ANS )
 		{
+		if ( ! c$dns?$query )
+			c$dns$query = ans$query;
+
 		c$dns$AA    = msg$AA;
 		c$dns$RA    = msg$RA;
 

scripts/base/protocols/ftp/files.bro

@@ -17,6 +17,10 @@ export {
 
 	## Describe the file being transferred.
 	global describe_file: function(f: fa_file): string;
+
+	redef record fa_file += { 
+		ftp: FTP::Info &optional;
+	};
 }
 
 function get_file_handle(c: connection, is_orig: bool): string
@@ -48,7 +52,6 @@ event bro_init() &priority=5
 	                          $describe        = FTP::describe_file]);
 	}
 
-
 event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priority=5
 	{
 	if ( [c$id$resp_h, c$id$resp_p] !in ftp_data_expected ) 
@@ -56,6 +59,17 @@ event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priori
 
 	local ftp = ftp_data_expected[c$id$resp_h, c$id$resp_p];
 	ftp$fuid = f$id;
-	if ( f?$mime_type )
-		ftp$mime_type = f$mime_type;
+
+	f$ftp = ftp;
+	}
+
+event file_sniff(f: fa_file, meta: fa_metadata) &priority=5
+	{
+	if ( ! f?$ftp )
+		return;
+
+	if ( ! meta?$mime_type )
+		return;
+
+	f$ftp$mime_type = meta$mime_type;
 	}

scripts/base/protocols/ftp/gridftp.bro

@@ -11,13 +11,13 @@
 ##! GridFTP data channels are identified by a heuristic that relies on
 ##! the fact that default settings for GridFTP clients typically
 ##! mutually authenticate the data channel with TLS/SSL and negotiate a
-##! NULL bulk cipher (no encryption).  Connections with those
-##! attributes are then polled for two minutes with decreasing frequency
-##! to check if the transfer sizes are large enough to indicate a
-##! GridFTP data channel that would be undesirable to analyze further
-##! (e.g. stop TCP reassembly).  A side effect is that true connection
-##! sizes are not logged, but at the benefit of saving CPU cycles that
-##! would otherwise go to analyzing the large (and likely benign) connections.
+##! NULL bulk cipher (no encryption). Connections with those attributes
+##! are marked as GridFTP if the data transfer within the first two minutes
+##! is big enough to indicate a GripFTP data channel that would be
+##! undesirable to analyze further (e.g. stop TCP reassembly).  A side
+##! effect is that true connection sizes are not logged, but at the benefit
+##! of saving CPU cycles that would otherwise go to analyzing the large
+##! (and likely benign) connections.
 
 @load ./info
 @load ./main
@@ -32,23 +32,14 @@ export {
 	## GridFTP data channel.
 	const size_threshold = 1073741824 &redef;
 
-	## Max number of times to check whether a connection's size exceeds the
+	## Time during which we check whether a connection's size exceeds the
 	## :bro:see:`GridFTP::size_threshold`.
-	const max_poll_count = 15 &redef;
+	const max_time = 2 min &redef;
 
 	## Whether to skip further processing of the GridFTP data channel once
 	## detected, which may help performance.
 	const skip_data = T &redef;
 
-	## Base amount of time between checking whether a GridFTP data connection
-	## has transferred more than :bro:see:`GridFTP::size_threshold` bytes.
-	const poll_interval = 1sec &redef;
-
-	## The amount of time the base :bro:see:`GridFTP::poll_interval` is
-	## increased by each poll interval.  Can be used to make more frequent
-	## checks at the start of a connection and gradually slow down.
-	const poll_interval_increase = 1sec &redef;
-
 	## Raised when a GridFTP data channel is detected.
 	##
 	## c: The connection pertaining to the GridFTP data channel.
@@ -79,23 +70,27 @@ event ftp_request(c: connection, command: string, arg: string) &priority=4
 		c$ftp$last_auth_requested = arg;
 	}
 
-function size_callback(c: connection, cnt: count): interval
+event ConnThreshold::bytes_threshold_crossed(c: connection, threshold: count, is_orig: bool)
 	{
-	if ( c$orig$size > size_threshold || c$resp$size > size_threshold )
+	if ( threshold < size_threshold || "gridftp-data" in c$service || c$duration > max_time )
+		return;
+
+	add c$service["gridftp-data"];
+	event GridFTP::data_channel_detected(c);
+
+	if ( skip_data )
+		skip_further_processing(c$id);
+	}
+
+event gridftp_possibility_timeout(c: connection)
+	{
+	# only remove if we did not already detect it and the connection
+	# is not yet at its end.
+	if ( "gridftp-data" !in c$service && ! (c?$conn && c$conn?$service) )
 		{
-		add c$service["gridftp-data"];
-		event GridFTP::data_channel_detected(c);
-
-		if ( skip_data )
-			skip_further_processing(c$id);
-
-		return -1sec;
+		ConnThreshold::delete_bytes_threshold(c, size_threshold, T);
+		ConnThreshold::delete_bytes_threshold(c, size_threshold, F);
 		}
-
-	if ( cnt >= max_poll_count )
-		return -1sec;
-
-	return poll_interval + poll_interval_increase * cnt;
 	}
 
 event ssl_established(c: connection) &priority=5
@@ -118,5 +113,9 @@ event ssl_established(c: connection) &priority=-3
 	# By default GridFTP data channels do mutual authentication and
 	# negotiate a cipher suite with a NULL bulk cipher.
 	if ( data_channel_initial_criteria(c) )
-		ConnPolling::watch(c, size_callback, 0, 0secs);
+		{
+		ConnThreshold::set_bytes_threshold(c, size_threshold, T);
+		ConnThreshold::set_bytes_threshold(c, size_threshold, F);
+		schedule max_time { gridftp_possibility_timeout(c) };
+		}
 	}

scripts/base/protocols/ftp/main.bro

@@ -52,7 +52,7 @@ redef likely_server_ports += { ports };
 
 event bro_init() &priority=5
 	{
-	Log::create_stream(FTP::LOG, [$columns=Info, $ev=log_ftp]);
+	Log::create_stream(FTP::LOG, [$columns=Info, $ev=log_ftp, $path="ftp"]);
 	Analyzer::register_for_ports(Analyzer::ANALYZER_FTP, ports);
 	}
 
@@ -274,7 +274,7 @@ event file_transferred(c: connection, prefix: string, descr: string,
 	if ( [id$resp_h, id$resp_p] in ftp_data_expected )
 		{
 		local s = ftp_data_expected[id$resp_h, id$resp_p];
-		s$mime_type = split1(mime_type, /;/)[1];
+		s$mime_type = split_string1(mime_type, /;/)[0];
 		}
 	}
 

scripts/base/protocols/http/entities.bro

@@ -35,11 +35,15 @@ export {
 		## body.
 		resp_mime_depth: count            &default=0;
 	};
+
+	redef record fa_file += {
+		http: HTTP::Info &optional;
+	};
 }
 
 event http_begin_entity(c: connection, is_orig: bool) &priority=10
 	{
-	set_state(c, F, is_orig);
+	set_state(c, is_orig);
 
 	if ( is_orig )
 		++c$http$orig_mime_depth;
@@ -67,6 +71,8 @@ event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priori
 	{
 	if ( f$source == "HTTP" && c?$http ) 
 		{
+		f$http = c$http;
+
 		if ( c$http?$current_entity && c$http$current_entity?$filename )
 			f$info$filename = c$http$current_entity$filename;
 
@@ -76,14 +82,6 @@ event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priori
 				c$http$orig_fuids = string_vec(f$id);
 			else
 				c$http$orig_fuids[|c$http$orig_fuids|] = f$id;
-
-			if ( f?$mime_type )
-				{
-				if ( ! c$http?$orig_mime_types )
-					c$http$orig_mime_types = string_vec(f$mime_type);
-				else
-					c$http$orig_mime_types[|c$http$orig_mime_types|] = f$mime_type;
-				}
 			}
 		else
 			{
@@ -91,17 +89,32 @@ event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priori
 				c$http$resp_fuids = string_vec(f$id);
 			else
 				c$http$resp_fuids[|c$http$resp_fuids|] = f$id;
-
-			if ( f?$mime_type )
-				{
-				if ( ! c$http?$resp_mime_types )
-					c$http$resp_mime_types = string_vec(f$mime_type);
-				else
-					c$http$resp_mime_types[|c$http$resp_mime_types|] = f$mime_type;
-				}
 			}
 		}
+	}
 
+event file_sniff(f: fa_file, meta: fa_metadata) &priority=5
+	{
+	if ( ! f?$http || ! f?$is_orig )
+		return;
+
+	if ( ! meta?$mime_type )
+		return;
+
+	if ( f$is_orig )
+		{
+		if ( ! f$http?$orig_mime_types )
+			f$http$orig_mime_types = string_vec(meta$mime_type);
+		else
+			f$http$orig_mime_types[|f$http$orig_mime_types|] = meta$mime_type;
+		}
+	else
+		{
+		if ( ! f$http?$resp_mime_types )
+			f$http$resp_mime_types = string_vec(meta$mime_type);
+		else
+			f$http$resp_mime_types[|f$http$resp_mime_types|] = meta$mime_type;
+		}
 	}
 
 event http_end_entity(c: connection, is_orig: bool) &priority=5

scripts/base/protocols/http/main.bro

@@ -89,6 +89,10 @@ export {
 		current_request:  count                &default=0;
 		## Current response in the pending queue.
 		current_response: count                &default=0;
+		## Track the current deepest transaction.
+		## This is meant to cope with missing requests
+		## and responses.
+		trans_depth:      count                &default=0;
 	};
 
 	## A list of HTTP headers typically used to indicate proxied requests.
@@ -135,7 +139,7 @@ redef likely_server_ports += { ports };
 # Initialize the HTTP logging stream and ports.
 event bro_init() &priority=5
 	{
-	Log::create_stream(HTTP::LOG, [$columns=Info, $ev=log_http]);
+	Log::create_stream(HTTP::LOG, [$columns=Info, $ev=log_http, $path="http"]);
 	Analyzer::register_for_ports(Analyzer::ANALYZER_HTTP, ports);
 	}
 
@@ -150,13 +154,11 @@ function new_http_session(c: connection): Info
 	tmp$ts=network_time();
 	tmp$uid=c$uid;
 	tmp$id=c$id;
-	# $current_request is set prior to the Info record creation so we
-	# can use the value directly here.
-	tmp$trans_depth = c$http_state$current_request;
+	tmp$trans_depth = ++c$http_state$trans_depth;
 	return tmp;
 	}
 
-function set_state(c: connection, request: bool, is_orig: bool)
+function set_state(c: connection, is_orig: bool)
 	{
 	if ( ! c?$http_state )
 		{
@@ -165,15 +167,20 @@ function set_state(c: connection, request: bool, is_orig: bool)
 		}
 
 	# These deal with new requests and responses.
-	if ( request || c$http_state$current_request !in c$http_state$pending )
-		c$http_state$pending[c$http_state$current_request] = new_http_session(c);
-	if ( ! is_orig && c$http_state$current_response !in c$http_state$pending )
-		c$http_state$pending[c$http_state$current_response] = new_http_session(c);
-
 	if ( is_orig )
+		{
+		if ( c$http_state$current_request !in c$http_state$pending )
+			c$http_state$pending[c$http_state$current_request] = new_http_session(c);
+
 		c$http = c$http_state$pending[c$http_state$current_request];
+		}
 	else
+		{
+		if ( c$http_state$current_response !in c$http_state$pending )
+			c$http_state$pending[c$http_state$current_response] = new_http_session(c);
+
 		c$http = c$http_state$pending[c$http_state$current_response];
+		}
 	}
 
 event http_request(c: connection, method: string, original_URI: string,
@@ -186,7 +193,7 @@ event http_request(c: connection, method: string, original_URI: string,
 		}
 
 	++c$http_state$current_request;
-	set_state(c, T, T);
+	set_state(c, T);
 
 	c$http$method = method;
 	c$http$uri = unescaped_URI;
@@ -208,8 +215,10 @@ event http_reply(c: connection, version: string, code: count, reason: string) &p
 	if ( c$http_state$current_response !in c$http_state$pending ||
 	     (c$http_state$pending[c$http_state$current_response]?$status_code &&
 	       ! code_in_range(c$http_state$pending[c$http_state$current_response]$status_code, 100, 199)) )
+		{
 		++c$http_state$current_response;
-	set_state(c, F, F);
+		}
+	set_state(c, F);
 
 	c$http$status_code = code;
 	c$http$status_msg = reason;
@@ -233,7 +242,7 @@ event http_reply(c: connection, version: string, code: count, reason: string) &p
 
 event http_header(c: connection, is_orig: bool, name: string, value: string) &priority=5
 	{
-	set_state(c, F, is_orig);
+	set_state(c, is_orig);
 
 	if ( is_orig ) # client headers
 		{
@@ -242,7 +251,7 @@ event http_header(c: connection, is_orig: bool, name: string, value: string) &pr
 
 		else if ( name == "HOST" )
 			# The split is done to remove the occasional port value that shows up here.
-			c$http$host = split1(value, /:/)[1];
+			c$http$host = split_string1(value, /:/)[0];
 
 		else if ( name == "RANGE" )
 			c$http$range_request = T;
@@ -257,17 +266,17 @@ event http_header(c: connection, is_orig: bool, name: string, value: string) &pr
 				add c$http$proxied[fmt("%s -> %s", name, value)];
 				}
 
-		else if ( name == "AUTHORIZATION" )
+		else if ( name == "AUTHORIZATION" || name == "PROXY-AUTHORIZATION" )
 			{
 			if ( /^[bB][aA][sS][iI][cC] / in value )
 				{
-				local userpass = decode_base64(sub(value, /[bB][aA][sS][iI][cC][[:blank:]]/, ""));
-				local up = split(userpass, /:/);
+				local userpass = decode_base64_conn(c$id, sub(value, /[bB][aA][sS][iI][cC][[:blank:]]/, ""));
+				local up = split_string(userpass, /:/);
 				if ( |up| >= 2 )
 					{
-					c$http$username = up[1];
+					c$http$username = up[0];
 					if ( c$http$capture_password )
-						c$http$password = up[2];
+						c$http$password = up[1];
 					}
 				else
 					{
@@ -278,12 +287,11 @@ event http_header(c: connection, is_orig: bool, name: string, value: string) &pr
 				}
 			}
 		}
-
 	}
 
 event http_message_done(c: connection, is_orig: bool, stat: http_message_stat) &priority = 5
 	{
-	set_state(c, F, is_orig);
+	set_state(c, is_orig);
 
 	if ( is_orig )
 		c$http$request_body_len = stat$body_length;

scripts/base/protocols/http/utils.bro

@@ -42,12 +42,12 @@ function extract_keys(data: string, kv_splitter: pattern): string_vec
 	{
 	local key_vec: vector of string = vector();
 	
-	local parts = split(data, kv_splitter);
+	local parts = split_string(data, kv_splitter);
 	for ( part_index in parts )
 		{
-		local key_val = split1(parts[part_index], /=/);
-		if ( 1 in key_val )
-			key_vec[|key_vec|] = key_val[1];
+		local key_val = split_string1(parts[part_index], /=/);
+		if ( 0 in key_val )
+			key_vec[|key_vec|] = key_val[0];
 		}
 	return key_vec;
 	}

scripts/base/protocols/irc/files.bro

@@ -12,6 +12,10 @@ export {
 
 	## Default file handle provider for IRC.
 	global get_file_handle: function(c: connection, is_orig: bool): string;
+
+	redef record fa_file += {
+		irc: IRC::Info &optional;
+	};
 }
 
 function get_file_handle(c: connection, is_orig: bool): string
@@ -34,6 +38,12 @@ event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priori
 	irc$fuid = f$id;
 	if ( irc?$dcc_file_name )
 		f$info$filename = irc$dcc_file_name;
-	if ( f?$mime_type )
-		irc$dcc_mime_type = f$mime_type;
+
+	f$irc = irc;
+	}
+
+event file_sniff(f: fa_file, meta: fa_metadata) &priority=5
+	{
+	if ( f?$irc && meta?$mime_type )
+		f$irc$dcc_mime_type = meta$mime_type;
 	}

scripts/base/protocols/irc/main.bro

@@ -43,7 +43,7 @@ redef likely_server_ports += { ports };
 
 event bro_init() &priority=5
 	{
-	Log::create_stream(IRC::LOG, [$columns=Info, $ev=irc_log]);
+	Log::create_stream(IRC::LOG, [$columns=Info, $ev=irc_log, $path="irc"]);
 	Analyzer::register_for_ports(Analyzer::ANALYZER_IRC, ports);
 	}
 	

scripts/base/protocols/krb/README

@@ -0,0 +1 @@
+Support for Kerberos protocol analysis.

scripts/base/protocols/krb/__load__.bro

@@ -0,0 +1,3 @@
+@load ./main
+@load ./files
+@load-sigs ./dpd.sig

scripts/base/protocols/krb/consts.bro

@@ -0,0 +1,99 @@
+module KRB;
+
+export {
+
+	const error_msg: table[count] of string = {
+		[0] = "KDC_ERR_NONE",
+		[1] = "KDC_ERR_NAME_EXP",
+		[2] = "KDC_ERR_SERVICE_EXP",
+		[3] = "KDC_ERR_BAD_PVNO",
+		[4] = "KDC_ERR_C_OLD_MAST_KVNO",
+		[5] = "KDC_ERR_S_OLD_MAST_KVNO",
+		[6] = "KDC_ERR_C_PRINCIPAL_UNKNOWN",
+		[7] = "KDC_ERR_S_PRINCIPAL_UNKNOWN",
+		[8] = "KDC_ERR_PRINCIPAL_NOT_UNIQUE",
+		[9] = "KDC_ERR_NULL_KEY",
+		[10] = "KDC_ERR_CANNOT_POSTDATE",
+		[11] = "KDC_ERR_NEVER_VALID",
+		[12] = "KDC_ERR_POLICY",
+		[13] = "KDC_ERR_BADOPTION",
+		[14] = "KDC_ERR_ETYPE_NOSUPP",
+		[15] = "KDC_ERR_SUMTYPE_NOSUPP",
+		[16] = "KDC_ERR_PADATA_TYPE_NOSUPP",
+		[17] = "KDC_ERR_TRTYPE_NOSUPP",
+		[18] = "KDC_ERR_CLIENT_REVOKED",
+		[19] = "KDC_ERR_SERVICE_REVOKED",
+		[20] = "KDC_ERR_TGT_REVOKED",
+		[21] = "KDC_ERR_CLIENT_NOTYET",
+		[22] = "KDC_ERR_SERVICE_NOTYET",
+		[23] = "KDC_ERR_KEY_EXPIRED",
+		[24] = "KDC_ERR_PREAUTH_FAILED",
+		[25] = "KDC_ERR_PREAUTH_REQUIRED",
+		[26] = "KDC_ERR_SERVER_NOMATCH",
+		[27] = "KDC_ERR_MUST_USE_USER2USER",
+		[28] = "KDC_ERR_PATH_NOT_ACCEPTED",
+		[29] = "KDC_ERR_SVC_UNAVAILABLE",
+		[31] = "KRB_AP_ERR_BAD_INTEGRITY",
+		[32] = "KRB_AP_ERR_TKT_EXPIRED",
+		[33] = "KRB_AP_ERR_TKT_NYV",
+		[34] = "KRB_AP_ERR_REPEAT",
+		[35] = "KRB_AP_ERR_NOT_US",
+		[36] = "KRB_AP_ERR_BADMATCH",
+		[37] = "KRB_AP_ERR_SKEW",
+		[38] = "KRB_AP_ERR_BADADDR",
+		[39] = "KRB_AP_ERR_BADVERSION",
+		[40] = "KRB_AP_ERR_MSG_TYPE",
+		[41] = "KRB_AP_ERR_MODIFIED",
+		[42] = "KRB_AP_ERR_BADORDER",
+		[44] = "KRB_AP_ERR_BADKEYVER",
+		[45] = "KRB_AP_ERR_NOKEY",
+		[46] = "KRB_AP_ERR_MUT_FAIL",
+		[47] = "KRB_AP_ERR_BADDIRECTION",
+		[48] = "KRB_AP_ERR_METHOD",
+		[49] = "KRB_AP_ERR_BADSEQ",
+		[50] = "KRB_AP_ERR_INAPP_CKSUM",
+		[51] = "KRB_AP_PATH_NOT_ACCEPTED",
+		[52] = "KRB_ERR_RESPONSE_TOO_BIG",
+		[60] = "KRB_ERR_GENERIC",
+		[61] = "KRB_ERR_FIELD_TOOLONG",
+		[62] = "KDC_ERROR_CLIENT_NOT_TRUSTED",
+		[63] = "KDC_ERROR_KDC_NOT_TRUSTED",
+		[64] = "KDC_ERROR_INVALID_SIG",
+		[65] = "KDC_ERR_KEY_TOO_WEAK",
+		[66] = "KDC_ERR_CERTIFICATE_MISMATCH",
+		[67] = "KRB_AP_ERR_NO_TGT",
+		[68] = "KDC_ERR_WRONG_REALM",
+		[69] = "KRB_AP_ERR_USER_TO_USER_REQUIRED",
+		[70] = "KDC_ERR_CANT_VERIFY_CERTIFICATE",
+		[71] = "KDC_ERR_INVALID_CERTIFICATE",
+		[72] = "KDC_ERR_REVOKED_CERTIFICATE",
+		[73] = "KDC_ERR_REVOCATION_STATUS_UNKNOWN",
+		[74] = "KDC_ERR_REVOCATION_STATUS_UNAVAILABLE",
+		[75] = "KDC_ERR_CLIENT_NAME_MISMATCH",
+		[76] = "KDC_ERR_KDC_NAME_MISMATCH",
+	};
+
+	const cipher_name: table[count] of string = {
+		[1] = "des-cbc-crc",
+		[2] = "des-cbc-md4",
+		[3] = "des-cbc-md5",
+		[5] = "des3-cbc-md5",
+		[7] = "des3-cbc-sha1",
+		[9] = "dsaWithSHA1-CmsOID",
+		[10] = "md5WithRSAEncryption-CmsOID",
+		[11] = "sha1WithRSAEncryption-CmsOID",
+		[12] = "rc2CBC-EnvOID",
+		[13] = "rsaEncryption-EnvOID",
+		[14] = "rsaES-OAEP-ENV-OID",
+		[15] = "des-ede3-cbc-Env-OID",
+		[16] = "des3-cbc-sha1-kd",
+		[17] = "aes128-cts-hmac-sha1-96",
+		[18] = "aes256-cts-hmac-sha1-96",
+		[23] = "rc4-hmac",
+		[24] = "rc4-hmac-exp",
+		[25] = "camellia128-cts-cmac",
+		[26] = "camellia256-cts-cmac",
+		[65] = "subkey-keymaterial",
+	};
+
+}

scripts/base/protocols/krb/dpd.sig

@@ -0,0 +1,26 @@
+# This is the ASN.1 encoded version and message type headers
+
+signature dpd_krb_udp_requests {
+	ip-proto == udp
+	payload /(\x6a|\x6c).{1,4}\x30.{1,4}\xa1\x03\x02\x01\x05\xa2\x03\x02\x01/
+	enable "krb"
+}
+
+signature dpd_krb_udp_replies {
+	ip-proto == udp
+	payload /(\x6b|\x6d|\x7e).{1,4}\x30.{1,4}\xa0\x03\x02\x01\x05\xa1\x03\x02\x01/
+	enable "krb"
+}
+
+signature dpd_krb_tcp_requests {
+	ip-proto == tcp
+	payload /.{4}(\x6a|\x6c).{1,4}\x30.{1,4}\xa1\x03\x02\x01\x05\xa2\x03\x02\x01/
+	enable "krb_tcp"
+}
+
+signature dpd_krb_tcp_replies {
+	ip-proto == tcp
+	payload /.{4}(\x6b|\x6d|\x7e).{1,4}\x30.{1,4}\xa0\x03\x02\x01\x05\xa1\x03\x02\x01/
+	enable "krb_tcp"
+}
+

scripts/base/protocols/krb/files.bro

@@ -0,0 +1,142 @@
+@load ./main
+@load base/utils/conn-ids
+@load base/frameworks/files
+@load base/files/x509
+
+module KRB;
+
+export {
+	redef record Info += {
+		# Client certificate
+		client_cert:		Files::Info &optional;
+		# Subject of client certificate, if any
+		client_cert_subject:	string &log &optional;
+		# File unique ID of client cert, if any
+		client_cert_fuid:	string &log &optional;
+
+		# Server certificate
+		server_cert:		Files::Info &optional;
+		# Subject of server certificate, if any
+		server_cert_subject:	string &log &optional;
+		# File unique ID of server cert, if any
+		server_cert_fuid:	string &log &optional;
+	};
+
+	## Default file handle provider for KRB.
+	global get_file_handle: function(c: connection, is_orig: bool): string;
+
+	## Default file describer for KRB.
+	global describe_file: function(f: fa_file): string;
+}
+
+function get_file_handle(c: connection, is_orig: bool): string
+	{
+	# Unused.  File handles are generated in the analyzer.
+	return "";
+	}
+
+function describe_file(f: fa_file): string
+	{
+	if ( f$source != "KRB_TCP" && f$source != "KRB" )
+		return "";
+
+	if ( ! f?$info || ! f$info?$x509 || ! f$info$x509?$certificate )
+		return "";
+
+	# It is difficult to reliably describe a certificate - especially since
+	# we do not know when this function is called (hence, if the data structures
+	# are already populated).
+	#
+	# Just return a bit of our connection information and hope that that is good enough.
+	for ( cid in f$conns )
+		{
+		if ( f$conns[cid]?$krb )
+			{
+			local c = f$conns[cid];
+			return cat(c$id$resp_h, ":", c$id$resp_p);
+			}
+		}
+
+	return cat("Serial: ", f$info$x509$certificate$serial, " Subject: ",
+			       f$info$x509$certificate$subject, " Issuer: ",
+			       f$info$x509$certificate$issuer);
+	}
+
+event bro_init() &priority=5
+	{
+	Files::register_protocol(Analyzer::ANALYZER_KRB_TCP,
+	                         [$get_file_handle = KRB::get_file_handle,
+	                          $describe        = KRB::describe_file]);
+
+	Files::register_protocol(Analyzer::ANALYZER_KRB,
+	                         [$get_file_handle = KRB::get_file_handle,
+	                          $describe        = KRB::describe_file]);
+	}
+
+event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priority=5
+	{
+	if ( f$source != "KRB_TCP" && f$source != "KRB" )
+		return;
+
+	local info: Info;
+
+	if ( ! c?$krb )
+		{
+		info$ts  = network_time();
+		info$uid = c$uid;
+		info$id  = c$id;
+		}
+	else
+		info = c$krb;
+
+	if ( is_orig )
+		{
+		info$client_cert = f$info;
+		info$client_cert_fuid = f$id;
+		}
+	else
+		{
+		info$server_cert = f$info;
+		info$server_cert_fuid = f$id;
+		}
+
+	c$krb = info;
+
+	Files::add_analyzer(f, Files::ANALYZER_X509);
+	# Always calculate hashes. They are not necessary for base scripts
+	# but very useful for identification, and required for policy scripts
+	Files::add_analyzer(f, Files::ANALYZER_MD5);
+	Files::add_analyzer(f, Files::ANALYZER_SHA1);
+	}
+
+function fill_in_subjects(c: connection)
+	{
+	if ( !c?$krb )
+		return;
+
+	if ( c$krb?$client_cert && c$krb$client_cert?$x509 && c$krb$client_cert$x509?$certificate )
+		c$krb$client_cert_subject = c$krb$client_cert$x509$certificate$subject;
+
+	if ( c$krb?$server_cert && c$krb$server_cert?$x509 && c$krb$server_cert$x509?$certificate )
+		c$krb$server_cert_subject = c$krb$server_cert$x509$certificate$subject;
+	}
+
+event krb_error(c: connection, msg: Error_Msg)
+	{
+	fill_in_subjects(c);
+	}
+
+event krb_as_response(c: connection, msg: KDC_Response)
+	{
+	fill_in_subjects(c);
+	}
+
+event krb_tgs_response(c: connection, msg: KDC_Response)
+	{
+	fill_in_subjects(c);
+	}
+
+event connection_state_remove(c: connection)
+	{
+	fill_in_subjects(c);
+	}

scripts/base/protocols/krb/main.bro

@@ -0,0 +1,251 @@
+##! Implements base functionality for KRB analysis. Generates the kerberos.log
+##! file.
+
+module KRB;
+
+@load ./consts
+
+export {
+	redef enum Log::ID += { LOG };
+
+	type Info: record {
+		## Timestamp for when the event happened.
+		ts:			time    &log;
+		## Unique ID for the connection.
+		uid:			string  &log;
+		## The connection's 4-tuple of endpoint addresses/ports.
+		id:			conn_id &log;
+
+		## Request type - Authentication Service ("AS") or
+		## Ticket Granting Service ("TGS")
+		request_type:		string &log &optional;
+		## Client
+		client:			string &log &optional;
+		## Service
+		service:		string &log;
+
+		## Request result
+		success:		bool &log &optional;
+		## Error code
+		error_code: 		count &optional;
+		## Error message
+		error_msg: 		string &log &optional;
+
+		## Ticket valid from
+		from:			time &log &optional;
+		## Ticket valid till
+		till:			time &log &optional;
+		## Ticket encryption type
+		cipher:			string &log &optional;
+
+		## Forwardable ticket requested
+		forwardable: 		bool &log &optional;
+		## Renewable ticket requested
+		renewable:		bool &log &optional;
+
+		## We've already logged this
+		logged: 		bool &default=F;
+	};
+
+	## The server response error texts which are *not* logged.
+	const ignored_errors: set[string] = {
+		# This will significantly increase the noisiness of the log.
+		# However, one attack is to iterate over principals, looking
+		# for ones that don't require preauth, and then performn
+		# an offline attack on that ticket. To detect that attack,
+		# log NEEDED_PREAUTH.
+		"NEEDED_PREAUTH",
+		# This is a more specific version of NEEDED_PREAUTH that's used
+		# by Windows AD Kerberos.
+		"Need to use PA-ENC-TIMESTAMP/PA-PK-AS-REQ",
+	} &redef;
+
+	## Event that can be handled to access the KRB record as it is sent on
+	## to the logging framework.
+	global log_krb: event(rec: Info);
+}
+
+redef record connection += {
+	krb: Info &optional;
+};
+
+const tcp_ports = { 88/tcp };
+const udp_ports = { 88/udp };
+redef likely_server_ports += { tcp_ports, udp_ports };
+
+event bro_init() &priority=5
+	{
+	Analyzer::register_for_ports(Analyzer::ANALYZER_KRB, udp_ports);
+	Analyzer::register_for_ports(Analyzer::ANALYZER_KRB_TCP, tcp_ports);
+	Log::create_stream(KRB::LOG, [$columns=Info, $ev=log_krb, $path="kerberos"]);
+	}
+
+event krb_error(c: connection, msg: Error_Msg) &priority=5
+	{
+	local info: Info;
+
+	if ( msg?$error_text && msg$error_text in ignored_errors )
+		{
+		if ( c?$krb ) delete c$krb;
+		return;
+		}
+
+	if ( c?$krb && c$krb$logged )
+		return;
+
+	if ( c?$krb )
+		info = c$krb;
+
+	if ( ! info?$ts )
+		{
+		info$ts  = network_time();
+		info$uid = c$uid;
+		info$id  = c$id;
+		}
+
+	if ( ! info?$client && ( msg?$client_name || msg?$client_realm ) )
+		info$client = fmt("%s%s", msg?$client_name ? msg$client_name + "/" : "",
+				 	  msg?$client_realm ? msg$client_realm : "");
+
+	info$service = msg$service_name;
+	info$success = F;
+
+	info$error_code = msg$error_code;
+
+	if ( msg?$error_text )			info$error_msg = msg$error_text;
+	else if ( msg$error_code in error_msg ) info$error_msg = error_msg[msg$error_code];
+
+	c$krb = info;
+	}
+
+event krb_error(c: connection, msg: Error_Msg) &priority=-5
+	{
+	if ( c?$krb )
+		{
+		Log::write(KRB::LOG, c$krb);
+		c$krb$logged = T;
+		}
+	}
+
+event krb_as_request(c: connection, msg: KDC_Request) &priority=5
+	{
+	if ( c?$krb && c$krb$logged )
+		return;
+
+	local info: Info;
+
+	if ( !c?$krb )
+		{
+		info$ts  = network_time();
+		info$uid = c$uid;
+		info$id  = c$id;
+		}
+	else
+		info = c$krb;
+
+	info$request_type = "AS";
+	info$client = fmt("%s/%s", msg$client_name, msg$service_realm);
+	info$service = msg$service_name;
+
+	if ( msg?$from )
+		info$from = msg$from;
+
+	info$till = msg$till;
+
+	info$forwardable = msg$kdc_options$forwardable;
+	info$renewable = msg$kdc_options$renewable;
+
+	c$krb = info;
+	}
+
+event krb_tgs_request(c: connection, msg: KDC_Request) &priority=5
+	{
+	if ( c?$krb && c$krb$logged )
+		return;
+
+	local info: Info;
+	info$ts  = network_time();
+	info$uid = c$uid;
+	info$id  = c$id;
+	info$request_type = "TGS";
+	info$service = msg$service_name;
+	if ( msg?$from ) info$from = msg$from;
+	info$till = msg$till;
+
+	info$forwardable = msg$kdc_options$forwardable;
+	info$renewable = msg$kdc_options$renewable;
+
+	c$krb = info;
+	}
+
+event krb_as_response(c: connection, msg: KDC_Response) &priority=5
+	{
+	local info: Info;
+
+	if ( c?$krb && c$krb$logged )
+		return;
+
+	if ( c?$krb )
+		info = c$krb;
+
+	if ( ! info?$ts )
+		{
+		info$ts  = network_time();
+		info$uid = c$uid;
+		info$id  = c$id;
+		}
+
+	if ( ! info?$client )
+		info$client = fmt("%s/%s", msg$client_name, msg$client_realm);
+
+	info$service = msg$ticket$service_name;
+	info$cipher  = cipher_name[msg$ticket$cipher];
+	info$success = T;
+
+	c$krb = info;
+	}
+
+event krb_as_response(c: connection, msg: KDC_Response) &priority=-5
+	{
+	Log::write(KRB::LOG, c$krb);
+	c$krb$logged = T;
+	}
+
+event krb_tgs_response(c: connection, msg: KDC_Response) &priority=5
+	{
+	local info: Info;
+
+	if ( c?$krb && c$krb$logged )
+		return;
+
+	if ( c?$krb )
+		info = c$krb;
+
+	if ( ! info?$ts )
+		{
+		info$ts  = network_time();
+		info$uid = c$uid;
+		info$id  = c$id;
+		}
+
+	if ( ! info?$client )
+		info$client = fmt("%s/%s", msg$client_name, msg$client_realm);
+
+	info$service = msg$ticket$service_name;
+	info$cipher  = cipher_name[msg$ticket$cipher];
+	info$success = T;
+
+	c$krb = info;
+	}
+
+event krb_tgs_response(c: connection, msg: KDC_Response) &priority=-5
+	{
+	Log::write(KRB::LOG, c$krb);
+	c$krb$logged = T;
+	}
+
+event connection_state_remove(c: connection) &priority=-5
+	{
+	if ( c?$krb && ! c$krb$logged )
+		Log::write(KRB::LOG, c$krb);
+	}

scripts/base/protocols/modbus/main.bro

@@ -34,7 +34,7 @@ redef likely_server_ports += { ports };
 
 event bro_init() &priority=5
 	{
-	Log::create_stream(Modbus::LOG, [$columns=Info, $ev=log_modbus]);
+	Log::create_stream(Modbus::LOG, [$columns=Info, $ev=log_modbus, $path="modbus"]);
 	Analyzer::register_for_ports(Analyzer::ANALYZER_MODBUS, ports);
 	}
 

scripts/base/protocols/mysql/README

@@ -0,0 +1 @@
+Support for MySQL protocol analysis.

scripts/base/protocols/mysql/__load__.bro

@@ -0,0 +1 @@
+@load ./main

scripts/base/protocols/mysql/consts.bro

@@ -0,0 +1,38 @@
+module MySQL;
+
+export {
+	const commands: table[count] of string = {
+		[0]  = "sleep",
+		[1]  = "quit",
+		[2]  = "init_db",
+		[3]  = "query",
+		[4]  = "field_list",
+		[5]  = "create_db",
+		[6]  = "drop_db",
+		[7]  = "refresh",
+		[8]  = "shutdown",
+		[9]  = "statistics",
+		[10] = "process_info",
+		[11] = "connect",
+		[12] = "process_kill",
+		[13] = "debug",
+		[14] = "ping",
+		[15] = "time",
+		[16] = "delayed_insert",
+		[17] = "change_user",
+		[18] = "binlog_dump",
+		[19] = "table_dump",
+		[20] = "connect_out",
+		[21] = "register_slave",
+		[22] = "stmt_prepare",
+		[23] = "stmt_execute",
+		[24] = "stmt_send_long_data",
+		[25] = "stmt_close",
+		[26] = "stmt_reset",
+		[27] = "set_option",
+		[28] = "stmt_fetch",
+		[29] = "daemon",
+		[30] = "binlog_dump_gtid",
+		[31] = "reset_connection",
+	} &default=function(i: count): string { return fmt("unknown-%d", i); };
+}

scripts/base/protocols/mysql/main.bro

@@ -0,0 +1,132 @@
+##! Implements base functionality for MySQL analysis. Generates the mysql.log file.
+
+module MySQL;
+
+@load ./consts
+
+export {
+	redef enum Log::ID += { mysql::LOG };
+
+	type Info: record {
+		## Timestamp for when the event happened.
+		ts:     time    &log;
+		## Unique ID for the connection.
+		uid:    string  &log;
+		## The connection's 4-tuple of endpoint addresses/ports.
+		id:     conn_id &log;
+		## The command that was issued
+		cmd:	string	&log;
+		## The argument issued to the command
+		arg:	string	&log;
+		## Did the server tell us that the command succeeded?
+		success: bool &log &optional;
+		## The number of affected rows, if any
+		rows: count &log &optional;
+		## Server message, if any
+		response: string &log &optional;
+	};
+
+	## Event that can be handled to access the MySQL record as it is sent on
+	## to the logging framework.
+	global log_mysql: event(rec: Info);
+}
+
+redef record connection += {
+	mysql: Info &optional;
+};
+
+const ports = { 1434/tcp, 3306/tcp };
+
+event bro_init() &priority=5
+	{
+	Log::create_stream(mysql::LOG, [$columns=Info, $ev=log_mysql, $path="mysql"]);
+	Analyzer::register_for_ports(Analyzer::ANALYZER_MYSQL, ports);
+	}
+
+event mysql_handshake(c: connection, username: string)
+	{
+	if ( ! c?$mysql )
+		{
+		local info: Info;
+		info$ts = network_time();
+		info$uid = c$uid;
+		info$id = c$id;
+		info$cmd = "login";
+		info$arg = username;
+		c$mysql = info;
+		}
+	}
+
+event mysql_command_request(c: connection, command: count, arg: string) &priority=5
+	{
+	if ( c?$mysql )
+		{
+		# We got a request, but we haven't logged our
+		# previous request yet, so let's do that now.
+		Log::write(mysql::LOG, c$mysql);
+		delete c$mysql;
+		}
+
+	local info: Info;
+	info$ts = network_time();
+	info$uid = c$uid;
+	info$id = c$id;
+	info$cmd = commands[command];
+	info$arg = sub(arg, /\0$/, "");
+	c$mysql = info;
+	}
+
+event mysql_command_request(c: connection, command: count, arg: string) &priority=-5
+	{
+	if ( c?$mysql && c$mysql?$cmd && c$mysql$cmd == "quit" )
+		{
+		# We get no response for quits, so let's just log it now.
+		Log::write(mysql::LOG, c$mysql);
+		delete c$mysql;
+		}
+	}
+
+event mysql_error(c: connection, code: count, msg: string) &priority=5
+	{
+	if ( c?$mysql )
+		{
+		c$mysql$success = F;
+		c$mysql$response = msg;
+		}
+	}
+
+event mysql_error(c: connection, code: count, msg: string) &priority=-5
+	{
+	if ( c?$mysql )
+		{
+		Log::write(mysql::LOG, c$mysql);
+		delete c$mysql;
+		}
+	}
+
+event mysql_ok(c: connection, affected_rows: count) &priority=5
+	{
+	if ( c?$mysql )
+		{
+		c$mysql$success = T;
+		c$mysql$rows = affected_rows;
+		}
+	}
+
+event mysql_ok(c: connection, affected_rows: count) &priority=-5
+	{
+	if ( c?$mysql )
+		{
+		Log::write(mysql::LOG, c$mysql);
+		delete c$mysql;
+		}
+	}
+
+event connection_state_remove(c: connection) &priority=-5
+	{
+	if ( c?$mysql )
+		{
+		Log::write(mysql::LOG, c$mysql);
+		delete c$mysql;
+		}
+	}

scripts/base/protocols/radius/README

@@ -0,0 +1 @@
+Support for RADIUS protocol analysis.

scripts/base/protocols/radius/main.bro

@@ -59,7 +59,7 @@ const ports = { 1812/udp };
 
 event bro_init() &priority=5
 	{
-	Log::create_stream(RADIUS::LOG, [$columns=Info, $ev=log_radius]);
+	Log::create_stream(RADIUS::LOG, [$columns=Info, $ev=log_radius, $path="radius"]);
 	Analyzer::register_for_ports(Analyzer::ANALYZER_RADIUS, ports);
 	}
 

scripts/base/protocols/rdp/README

@@ -0,0 +1 @@
+Support for Remote Desktop Protocol (RDP) analysis.

scripts/base/protocols/rdp/__load__.bro

@@ -0,0 +1,3 @@
+@load ./consts
+@load ./main
+@load-sigs ./dpd.sig

scripts/base/protocols/rdp/consts.bro

@@ -0,0 +1,323 @@
+module RDP;
+
+export {
+	# http://www.c-amie.co.uk/technical/mstsc-versions/
+	const builds = {
+		[0419] = "RDP 4.0",
+		[2195] = "RDP 5.0",
+		[2221] = "RDP 5.0",
+		[2600] = "RDP 5.1",
+		[3790] = "RDP 5.2",
+		[6000] = "RDP 6.0",
+		[6001] = "RDP 6.1",
+		[6002] = "RDP 6.2",
+		[7600] = "RDP 7.0",
+		[7601] = "RDP 7.1",
+		[9200] = "RDP 8.0",
+		[9600] = "RDP 8.1",
+		[25189] = "RDP 8.0 (Mac)",
+		[25282] = "RDP 8.0 (Mac)"
+	} &default = function(n: count): string { return fmt("client_build-%d", n); };
+
+	const security_protocols = {
+		[0x00] = "RDP",
+		[0x01] = "SSL",
+		[0x02] = "HYBRID",
+		[0x08] = "HYBRID_EX"
+	} &default = function(n: count): string { return fmt("security_protocol-%d", n); };
+
+	const failure_codes = {
+		[0x01] = "SSL_REQUIRED_BY_SERVER",
+		[0x02] = "SSL_NOT_ALLOWED_BY_SERVER",
+		[0x03] = "SSL_CERT_NOT_ON_SERVER",
+		[0x04] = "INCONSISTENT_FLAGS",
+		[0x05] = "HYBRID_REQUIRED_BY_SERVER",
+		[0x06] = "SSL_WITH_USER_AUTH_REQUIRED_BY_SERVER"
+	} &default = function(n: count): string { return fmt("failure_code-%d", n); };
+
+	const cert_types = {
+		[1] = "RSA",
+		[2] = "X.509"
+	} &default = function(n: count): string { return fmt("cert_type-%d", n); };
+
+	const encryption_methods = {
+		[0] = "None",
+		[1] = "40bit",
+		[2] = "128bit",
+		[8] = "56bit",
+		[10] = "FIPS"
+	} &default = function(n: count): string { return fmt("encryption_method-%d", n); };
+
+	const encryption_levels = {
+		[0] = "None",
+		[1] = "Low",
+		[2] = "Client compatible",
+		[3] = "High",
+		[4] = "FIPS"
+	} &default = function(n: count): string { return fmt("encryption_level-%d", n); };
+
+	const high_color_depths = {
+		[0x0004] = "4bit",
+		[0x0008] = "8bit",
+		[0x000F] = "15bit",
+		[0x0010] = "16bit",
+		[0x0018] = "24bit"
+	} &default = function(n: count): string { return fmt("high_color_depth-%d", n); };
+
+	const color_depths = {
+		[0x0001] = "24bit",
+		[0x0002] = "16bit",
+		[0x0004] = "15bit",
+		[0x0008] = "32bit"
+	} &default = function(n: count): string { return fmt("color_depth-%d", n); };
+
+	const results = {
+		[0] = "Success",
+		[1] = "User rejected",
+		[2] = "Resources not available",
+		[3] = "Rejected for symmetry breaking",
+		[4] = "Locked conference",
+	} &default = function(n: count): string { return fmt("result-%d", n); };
+
+	# http://msdn.microsoft.com/en-us/goglobal/bb964664.aspx
+	const languages = {
+		[1078] = "Afrikaans - South Africa",
+		[1052] = "Albanian - Albania",
+		[1156] = "Alsatian",
+		[1118] = "Amharic - Ethiopia",
+		[1025] = "Arabic - Saudi Arabia",
+		[5121] = "Arabic - Algeria",
+		[15361] = "Arabic - Bahrain",
+		[3073] = "Arabic - Egypt",
+		[2049] = "Arabic - Iraq",
+		[11265] = "Arabic - Jordan",
+		[13313] = "Arabic - Kuwait",
+		[12289] = "Arabic - Lebanon",
+		[4097] = "Arabic - Libya",
+		[6145] = "Arabic - Morocco",
+		[8193] = "Arabic - Oman",
+		[16385] = "Arabic - Qatar",
+		[10241] = "Arabic - Syria",
+		[7169] = "Arabic - Tunisia",
+		[14337] = "Arabic - U.A.E.",
+		[9217] = "Arabic - Yemen",
+		[1067] = "Armenian - Armenia",
+		[1101] = "Assamese",
+		[2092] = "Azeri (Cyrillic)",
+		[1068] = "Azeri (Latin)",
+		[1133] = "Bashkir",
+		[1069] = "Basque",
+		[1059] = "Belarusian",
+		[1093] = "Bengali (India)",
+		[2117] = "Bengali (Bangladesh)",
+		[5146] = "Bosnian (Bosnia/Herzegovina)",
+		[1150] = "Breton",
+		[1026] = "Bulgarian",
+		[1109] = "Burmese",
+		[1027] = "Catalan",
+		[1116] = "Cherokee - United States",
+		[2052] = "Chinese - People's Republic of China",
+		[4100] = "Chinese - Singapore",
+		[1028] = "Chinese - Taiwan",
+		[3076] = "Chinese - Hong Kong SAR",
+		[5124] = "Chinese - Macao SAR",
+		[1155] = "Corsican",
+		[1050] = "Croatian",
+		[4122] = "Croatian (Bosnia/Herzegovina)",
+		[1029] = "Czech",
+		[1030] = "Danish",
+		[1164] = "Dari",
+		[1125] = "Divehi",
+		[1043] = "Dutch - Netherlands",
+		[2067] = "Dutch - Belgium",
+		[1126] = "Edo",
+		[1033] = "English - United States",
+		[2057] = "English - United Kingdom",
+		[3081] = "English - Australia",
+		[10249] = "English - Belize",
+		[4105] = "English - Canada",
+		[9225] = "English - Caribbean",
+		[15369] = "English - Hong Kong SAR",
+		[16393] = "English - India",
+		[14345] = "English - Indonesia",
+		[6153] = "English - Ireland",
+		[8201] = "English - Jamaica",
+		[17417] = "English - Malaysia",
+		[5129] = "English - New Zealand",
+		[13321] = "English - Philippines",
+		[18441] = "English - Singapore",
+		[7177] = "English - South Africa",
+		[11273] = "English - Trinidad",
+		[12297] = "English - Zimbabwe",
+		[1061] = "Estonian",
+		[1080] = "Faroese",
+		[1065] = "Farsi",
+		[1124] = "Filipino",
+		[1035] = "Finnish",
+		[1036] = "French - France",
+		[2060] = "French - Belgium",
+		[11276] = "French - Cameroon",
+		[3084] = "French - Canada",
+		[9228] = "French - Democratic Rep. of Congo",
+		[12300] = "French - Cote d'Ivoire",
+		[15372] = "French - Haiti",
+		[5132] = "French - Luxembourg",
+		[13324] = "French - Mali",
+		[6156] = "French - Monaco",
+		[14348] = "French - Morocco",
+		[58380] = "French - North Africa",
+		[8204] = "French - Reunion",
+		[10252] = "French - Senegal",
+		[4108] = "French - Switzerland",
+		[7180] = "French - West Indies",
+		[1122] = "French - West Indies",
+		[1127] = "Fulfulde - Nigeria",
+		[1071] = "FYRO Macedonian",
+		[1110] = "Galician",
+		[1079] = "Georgian",
+		[1031] = "German - Germany",
+		[3079] = "German - Austria",
+		[5127] = "German - Liechtenstein",
+		[4103] = "German - Luxembourg",
+		[2055] = "German - Switzerland",
+		[1032] = "Greek",
+		[1135] = "Greenlandic",
+		[1140] = "Guarani - Paraguay",
+		[1095] = "Gujarati",
+		[1128] = "Hausa - Nigeria",
+		[1141] = "Hawaiian - United States",
+		[1037] = "Hebrew",
+		[1081] = "Hindi",
+		[1038] = "Hungarian",
+		[1129] = "Ibibio - Nigeria",
+		[1039] = "Icelandic",
+		[1136] = "Igbo - Nigeria",
+		[1057] = "Indonesian",
+		[1117] = "Inuktitut",
+		[2108] = "Irish",
+		[1040] = "Italian - Italy",
+		[2064] = "Italian - Switzerland",
+		[1041] = "Japanese",
+		[1158] = "K'iche",
+		[1099] = "Kannada",
+		[1137] = "Kanuri - Nigeria",
+		[2144] = "Kashmiri",
+		[1120] = "Kashmiri (Arabic)",
+		[1087] = "Kazakh",
+		[1107] = "Khmer",
+		[1159] = "Kinyarwanda",
+		[1111] = "Konkani",
+		[1042] = "Korean",
+		[1088] = "Kyrgyz (Cyrillic)",
+		[1108] = "Lao",
+		[1142] = "Latin",
+		[1062] = "Latvian",
+		[1063] = "Lithuanian",
+		[1134] = "Luxembourgish",
+		[1086] = "Malay - Malaysia",
+		[2110] = "Malay - Brunei Darussalam",
+		[1100] = "Malayalam",
+		[1082] = "Maltese",
+		[1112] = "Manipuri",
+		[1153] = "Maori - New Zealand",
+		[1146] = "Mapudungun",
+		[1102] = "Marathi",
+		[1148] = "Mohawk",
+		[1104] = "Mongolian (Cyrillic)",
+		[2128] = "Mongolian (Mongolian)",
+		[1121] = "Nepali",
+		[2145] = "Nepali - India",
+		[1044] = "Norwegian (Bokmål)",
+		[2068] = "Norwegian (Nynorsk)",
+		[1154] = "Occitan",
+		[1096] = "Oriya",
+		[1138] = "Oromo",
+		[1145] = "Papiamentu",
+		[1123] = "Pashto",
+		[1045] = "Polish",
+		[1046] = "Portuguese - Brazil",
+		[2070] = "Portuguese - Portugal",
+		[1094] = "Punjabi",
+		[2118] = "Punjabi (Pakistan)",
+		[1131] = "Quecha - Bolivia",
+		[2155] = "Quecha - Ecuador",
+		[3179] = "Quecha - Peru	CB",
+		[1047] = "Rhaeto-Romanic",
+		[1048] = "Romanian",
+		[2072] = "Romanian - Moldava",
+		[1049] = "Russian",
+		[2073] = "Russian - Moldava",
+		[1083] = "Sami (Lappish)",
+		[1103] = "Sanskrit",
+		[1084] = "Scottish Gaelic",
+		[1132] = "Sepedi",
+		[3098] = "Serbian (Cyrillic)",
+		[2074] = "Serbian (Latin)",
+		[1113] = "Sindhi - India",
+		[2137] = "Sindhi - Pakistan",
+		[1115] = "Sinhalese - Sri Lanka",
+		[1051] = "Slovak",
+		[1060] = "Slovenian",
+		[1143] = "Somali",
+		[1070] = "Sorbian",
+		[3082] = "Spanish - Spain (Modern Sort)",
+		[1034] = "Spanish - Spain (Traditional Sort)",
+		[11274] = "Spanish - Argentina",
+		[16394] = "Spanish - Bolivia",
+		[13322] = "Spanish - Chile",
+		[9226] = "Spanish - Colombia",
+		[5130] = "Spanish - Costa Rica",
+		[7178] = "Spanish - Dominican Republic",
+		[12298] = "Spanish - Ecuador",
+		[17418] = "Spanish - El Salvador",
+		[4106] = "Spanish - Guatemala",
+		[18442] = "Spanish - Honduras",
+		[22538] = "Spanish - Latin America",
+		[2058] = "Spanish - Mexico",
+		[19466] = "Spanish - Nicaragua",
+		[6154] = "Spanish - Panama",
+		[15370] = "Spanish - Paraguay",
+		[10250] = "Spanish - Peru",
+		[20490] = "Spanish - Puerto Rico",
+		[21514] = "Spanish - United States",
+		[14346] = "Spanish - Uruguay",
+		[8202] = "Spanish - Venezuela",
+		[1072] = "Sutu",
+		[1089] = "Swahili",
+		[1053] = "Swedish",
+		[2077] = "Swedish - Finland",
+		[1114] = "Syriac",
+		[1064] = "Tajik",
+		[1119] = "Tamazight (Arabic)",
+		[2143] = "Tamazight (Latin)",
+		[1097] = "Tamil",
+		[1092] = "Tatar",
+		[1098] = "Telugu",
+		[1054] = "Thai",
+		[2129] = "Tibetan - Bhutan",
+		[1105] = "Tibetan - People's Republic of China",
+		[2163] = "Tigrigna - Eritrea",
+		[1139] = "Tigrigna - Ethiopia",
+		[1073] = "Tsonga",
+		[1074] = "Tswana",
+		[1055] = "Turkish",
+		[1090] = "Turkmen",
+		[1152] = "Uighur - China",
+		[1058] = "Ukrainian",
+		[1056] = "Urdu",
+		[2080] = "Urdu - India",
+		[2115] = "Uzbek (Cyrillic)",
+		[1091] = "Uzbek (Latin)",
+		[1075] = "Venda",
+		[1066] = "Vietnamese",
+		[1106] = "Welsh",
+		[1160] = "Wolof",
+		[1076] = "Xhosa",
+		[1157] = "Yakut",
+		[1144] = "Yi",
+		[1085] = "Yiddish",
+		[1130] = "Yoruba",
+		[1077] = "Zulu",
+		[1279] = "HID (Human Interface Device)",
+	} &default = function(n: count): string { return fmt("keyboard-%d", n); };
+}

scripts/base/protocols/rdp/dpd.sig

@@ -0,0 +1,12 @@
+signature dpd_rdp_client {
+	ip-proto == tcp
+	# Client request
+	payload /.*(Cookie: mstshash\=|Duca.*(rdpdr|rdpsnd|drdynvc|cliprdr))/
+	requires-reverse-signature dpd_rdp_server
+	enable "rdp"
+}
+
+signature dpd_rdp_server {
+	ip-proto == tcp
+	payload /(.{5}\xd0|.*McDn)/
+}

scripts/base/protocols/rdp/main.bro

@@ -0,0 +1,269 @@
+##! Implements base functionality for RDP analysis. Generates the rdp.log file.
+
+@load ./consts
+
+module RDP;
+
+export {
+	redef enum Log::ID += { LOG };
+
+	type Info: record {
+		## Timestamp for when the event happened.
+		ts:                    time    &log;
+		## Unique ID for the connection.
+		uid:                   string  &log;
+		## The connection's 4-tuple of endpoint addresses/ports.
+		id:                    conn_id &log;
+		## Cookie value used by the client machine.
+		## This is typically a username.
+		cookie:                string  &log &optional;
+		## Status result for the connection.  It's a mix between
+		## RDP negotation failure messages and GCC server create
+		## response messages.
+		result:                string  &log &optional;
+		## Security protocol chosen by the server.
+		security_protocol:     string &log &optional;
+
+		## Keyboard layout (language) of the client machine.
+		keyboard_layout:       string  &log &optional;
+		## RDP client version used by the client machine.
+		client_build:          string  &log &optional;
+		## Name of the client machine.
+		client_name:           string  &log &optional;
+		## Product ID of the client machine.
+		client_dig_product_id: string  &log &optional;
+		## Desktop width of the client machine.
+		desktop_width:         count   &log &optional;
+		## Desktop height of the client machine.
+		desktop_height:        count   &log &optional;
+		## The color depth requested by the client in 
+		## the high_color_depth field.
+		requested_color_depth: string  &log &optional;
+
+		## If the connection is being encrypted with native
+		## RDP encryption, this is the type of cert 
+		## being used.
+		cert_type:             string  &log &optional;
+		## The number of certs seen.  X.509 can transfer an 
+		## entire certificate chain.
+		cert_count:            count   &log &default=0;
+		## Indicates if the provided certificate or certificate
+		## chain is permanent or temporary.
+		cert_permanent:        bool    &log &optional;
+		## Encryption level of the connection.
+		encryption_level:      string  &log &optional;
+		## Encryption method of the connection. 
+		encryption_method:     string  &log &optional;
+		};
+
+	## If true, detach the RDP analyzer from the connection to prevent
+	## continuing to process encrypted traffic.
+	const disable_analyzer_after_detection = F &redef;
+
+	## The amount of time to monitor an RDP session from when it is first 
+	## identified. When this interval is reached, the session is logged.
+	const rdp_check_interval = 10secs &redef;
+
+	## Event that can be handled to access the rdp record as it is sent on
+	## to the logging framework.
+	global log_rdp: event(rec: Info);
+}
+
+# Internal fields that aren't useful externally
+redef record Info += {
+	## The analyzer ID used for the analyzer instance attached
+	## to each connection.  It is not used for logging since it's a
+	## meaningless arbitrary number.
+	analyzer_id: count &optional;
+	## Track status of logging RDP connections.
+	done:        bool  &default=F;
+};
+
+redef record connection += {
+	rdp: Info &optional;
+};
+
+const ports = { 3389/tcp };
+redef likely_server_ports += { ports };
+
+event bro_init() &priority=5
+	{
+	Log::create_stream(RDP::LOG, [$columns=RDP::Info, $ev=log_rdp, $path="rdp"]);
+	Analyzer::register_for_ports(Analyzer::ANALYZER_RDP, ports);
+	}
+
+function write_log(c: connection)
+	{
+	local info = c$rdp;
+
+	if ( info$done )
+		return;
+
+	# Mark this record as fully logged and finished.
+	info$done = T;
+
+	# Verify that the RDP session contains
+	# RDP data before writing it to the log. 
+	if ( info?$cookie || info?$keyboard_layout || info?$result )
+		Log::write(RDP::LOG, info);
+	}
+
+event check_record(c: connection)
+	{
+	# If the record was logged, then stop processing.
+	if ( c$rdp$done )
+		return;
+
+	# If the value rdp_check_interval has passed since the 
+	# RDP session was started, then log the record. 
+	local diff = network_time() - c$rdp$ts;
+	if ( diff > rdp_check_interval )
+		{
+		write_log(c);
+
+		# Remove the analyzer if it is still attached.
+		if ( disable_analyzer_after_detection && 
+		     connection_exists(c$id) && 
+		     c$rdp?$analyzer_id )
+			{
+			disable_analyzer(c$id, c$rdp$analyzer_id);
+			}
+
+		return;
+		}
+	else
+		{
+		# If the analyzer is attached and the duration
+		# to monitor the RDP session was not met, then
+		# reschedule the logging event.
+		schedule rdp_check_interval { check_record(c) };
+		}
+	}
+
+function set_session(c: connection)
+	{
+	if ( ! c?$rdp )
+		{
+		c$rdp = [$ts=network_time(),$id=c$id,$uid=c$uid];
+		# The RDP session is scheduled to be logged from
+		# the time it is first initiated.
+		schedule rdp_check_interval { check_record(c) };
+		}
+	}
+
+event rdp_connect_request(c: connection, cookie: string) &priority=5
+	{
+	set_session(c);
+
+	c$rdp$cookie = cookie;
+	}
+
+event rdp_negotiation_response(c: connection, security_protocol: count) &priority=5
+	{
+	set_session(c);
+
+	c$rdp$security_protocol = security_protocols[security_protocol];
+	}
+
+event rdp_negotiation_failure(c: connection, failure_code: count) &priority=5
+	{
+	set_session(c);
+
+	c$rdp$result = failure_codes[failure_code];
+	}
+
+event rdp_client_core_data(c: connection, data: RDP::ClientCoreData) &priority=5
+	{
+	set_session(c);
+
+	c$rdp$keyboard_layout       = RDP::languages[data$keyboard_layout];
+	c$rdp$client_build          = RDP::builds[data$client_build];
+	c$rdp$client_name           = data$client_name;
+	c$rdp$client_dig_product_id = data$dig_product_id;
+	c$rdp$desktop_width         = data$desktop_width;
+	c$rdp$desktop_height        = data$desktop_height;
+
+	if ( data?$ec_flags && data$ec_flags$want_32bpp_session )
+		c$rdp$requested_color_depth = "32bit";
+	else
+		c$rdp$requested_color_depth = RDP::high_color_depths[data$high_color_depth];
+	}
+
+event rdp_gcc_server_create_response(c: connection, result: count) &priority=5
+	{
+	set_session(c);
+
+	c$rdp$result = RDP::results[result];
+	}
+
+event rdp_server_security(c: connection, encryption_method: count, encryption_level: count) &priority=5
+	{
+	set_session(c);
+
+	c$rdp$encryption_method = RDP::encryption_methods[encryption_method];
+	c$rdp$encryption_level = RDP::encryption_levels[encryption_level];
+	}
+
+event rdp_server_certificate(c: connection, cert_type: count, permanently_issued: bool) &priority=5
+	{
+	set_session(c);
+
+	c$rdp$cert_type = RDP::cert_types[cert_type];
+
+	# There are no events for proprietary/RSA certs right
+	# now so we manually count this one.
+	if ( c$rdp$cert_type == "RSA" )
+		++c$rdp$cert_count;
+	
+	c$rdp$cert_permanent = permanently_issued;
+	}
+
+event rdp_begin_encryption(c: connection, security_protocol: count) &priority=5
+	{
+	set_session(c);
+
+	if ( ! c$rdp?$result )
+		{
+		c$rdp$result = "encrypted";
+		}
+
+	c$rdp$security_protocol = security_protocols[security_protocol];
+	}
+
+event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priority=5
+	{
+	if ( c?$rdp && f$source == "RDP" )
+		{
+		# Count up X509 certs.
+		++c$rdp$cert_count;
+
+		Files::add_analyzer(f, Files::ANALYZER_X509);
+		Files::add_analyzer(f, Files::ANALYZER_MD5);
+		Files::add_analyzer(f, Files::ANALYZER_SHA1);
+		}
+	}
+
+event protocol_confirmation(c: connection, atype: Analyzer::Tag, aid: count) &priority=5
+	{
+	if ( atype == Analyzer::ANALYZER_RDP )
+		{
+		set_session(c);
+		c$rdp$analyzer_id = aid;
+		}
+	}
+
+event protocol_violation(c: connection, atype: Analyzer::Tag, aid: count, reason: string) &priority=5
+	{
+	# If a protocol violation occurs, then log the record immediately.
+	if ( c?$rdp )
+		write_log(c);
+	}
+
+event connection_state_remove(c: connection) &priority=-5
+	{
+	# If the connection is removed, then log the record immediately.
+	if ( c?$rdp )
+		{
+		write_log(c);
+		}
+	}

scripts/base/protocols/sip/README

@@ -0,0 +1 @@
+Support for Session Initiation Protocol (SIP) analysis.

scripts/base/protocols/sip/__load__.bro

@@ -0,0 +1,3 @@
+@load ./main
+
+@load-sigs ./dpd.sig

scripts/base/protocols/sip/dpd.sig

@@ -0,0 +1,19 @@
+signature dpd_sip_udp_req {
+  ip-proto == udp
+  payload /.* SIP\/[0-9]\.[0-9]\x0d\x0a/
+  enable "sip"
+}
+
+signature dpd_sip_udp_resp {
+  ip-proto == udp
+  payload /^ ?SIP\/[0-9]\.[0-9](\x0d\x0a| [0-9][0-9][0-9] )/
+  enable "sip"
+}
+
+# We don't support SIP-over-TCP yet.
+#
+# signature dpd_sip_tcp {
+#   ip-proto == tcp
+#   payload /^( SIP\/[0-9]\.[0-9]\x0d\x0a|SIP\/[0-9]\.[0-9] [0-9][0-9][0-9] )/
+#   enable "sip_tcp"
+# }

scripts/base/protocols/sip/main.bro

@@ -0,0 +1,301 @@
+##! Implements base functionality for SIP analysis.  The logging model is
+##! to log request/response pairs and all relevant metadata together in
+##! a single record.
+
+@load base/utils/numbers
+@load base/utils/files
+
+module SIP;
+
+export {
+	redef enum Log::ID += { LOG };
+
+	type Info: record {
+		## Timestamp for when the request happened.
+		ts:                      time              &log;
+		## Unique ID for the connection.
+		uid:                     string            &log;
+		## The connection's 4-tuple of endpoint addresses/ports.
+		id:                      conn_id           &log;
+		## Represents the pipelined depth into the connection of this
+		## request/response transaction.
+		trans_depth:             count             &log;
+		## Verb used in the SIP request (INVITE, REGISTER etc.).
+		method:                  string            &log &optional;
+		## URI used in the request.
+		uri:                     string            &log &optional;
+		## Contents of the Date: header from the client
+		date:                    string            &log &optional;
+		## Contents of the request From: header
+		## Note: The tag= value that's usually appended to the sender
+		## is stripped off and not logged.
+		request_from:            string            &log &optional;
+		## Contents of the To: header
+		request_to:              string            &log &optional;
+		## Contents of the response From: header
+		## Note: The ``tag=`` value that's usually appended to the sender
+		## is stripped off and not logged.
+		response_from:            string            &log &optional;
+		## Contents of the response To: header
+		response_to:              string            &log &optional;
+
+		## Contents of the Reply-To: header
+		reply_to:                string            &log &optional;
+		## Contents of the Call-ID: header from the client
+		call_id:                 string            &log &optional;
+		## Contents of the CSeq: header from the client
+		seq:                     string            &log &optional;
+		## Contents of the Subject: header from the client
+		subject:                 string            &log &optional;
+		## The client message transmission path, as extracted from the headers.
+		request_path:            vector of string  &log &optional;
+		## The server message transmission path, as extracted from the headers.
+		response_path:           vector of string  &log &optional;
+		## Contents of the User-Agent: header from the client
+		user_agent:              string            &log &optional;
+		## Status code returned by the server.
+		status_code:             count             &log &optional;
+		## Status message returned by the server.
+		status_msg:              string            &log &optional;
+		## Contents of the Warning: header
+		warning:                 string            &log &optional;
+		## Contents of the Content-Length: header from the client
+		request_body_len:        count             &log &optional;
+		## Contents of the Content-Length: header from the server
+		response_body_len:       count             &log &optional;
+		## Contents of the Content-Type: header from the server
+		content_type:            string            &log &optional;
+	};
+
+	type State: record {
+		## Pending requests.
+		pending:          table[count] of Info;
+		## Current request in the pending queue.
+		current_request:  count                &default=0;
+		## Current response in the pending queue.
+		current_response: count                &default=0;
+	};
+
+	## A list of SIP methods. Other methods will generate a weird. Note
+	## that the SIP analyzer will only accept methods consisting solely
+	## of letters ``[A-Za-z]``.
+	const sip_methods: set[string] = {
+		"REGISTER", "INVITE", "ACK", "CANCEL", "BYE", "OPTIONS"
+	} &redef;
+
+	## Event that can be handled to access the SIP record as it is sent on
+	## to the logging framework.
+	global log_sip: event(rec: Info);
+}
+
+# Add the sip state tracking fields to the connection record.
+redef record connection += {
+	sip:        Info  &optional;
+	sip_state:  State &optional;
+};
+
+const ports = { 5060/udp };
+redef likely_server_ports += { ports };
+
+event bro_init() &priority=5
+	{
+	Log::create_stream(SIP::LOG, [$columns=Info, $ev=log_sip, $path="sip"]);
+	Analyzer::register_for_ports(Analyzer::ANALYZER_SIP, ports);
+	}
+
+function new_sip_session(c: connection): Info
+	{
+	local tmp: Info;
+	tmp$ts=network_time();
+	tmp$uid=c$uid;
+	tmp$id=c$id;
+	# $current_request is set prior to the Info record creation so we
+	# can use the value directly here.
+	tmp$trans_depth = c$sip_state$current_request;
+
+	tmp$request_path = vector();
+	tmp$response_path = vector();
+
+	return tmp;
+	}
+
+function set_state(c: connection, is_request: bool)
+	{
+	if ( ! c?$sip_state )
+		{
+		local s: State;
+		c$sip_state = s;
+		}
+
+	if ( is_request )
+		{
+		if ( c$sip_state$current_request !in c$sip_state$pending )
+			c$sip_state$pending[c$sip_state$current_request] = new_sip_session(c);
+
+		c$sip = c$sip_state$pending[c$sip_state$current_request];
+		}
+	else
+		{
+		if ( c$sip_state$current_response !in c$sip_state$pending )
+			c$sip_state$pending[c$sip_state$current_response] = new_sip_session(c);
+
+		c$sip = c$sip_state$pending[c$sip_state$current_response];
+		}
+	}
+
+function flush_pending(c: connection)
+	{
+	# Flush all pending but incomplete request/response pairs.
+	if ( c?$sip_state )
+		{
+		for ( r in c$sip_state$pending )
+			{
+			# We don't use pending elements at index 0.
+			if ( r == 0 )
+				next;
+			
+			Log::write(SIP::LOG, c$sip_state$pending[r]);
+			}
+		}
+	}
+
+event sip_request(c: connection, method: string, original_URI: string, version: string) &priority=5
+	{
+	set_state(c, T);
+
+	c$sip$method = method;
+	c$sip$uri = original_URI;
+
+	if ( method !in sip_methods )
+		event conn_weird("unknown_SIP_method", c, method);
+	}
+
+event sip_reply(c: connection, version: string, code: count, reason: string) &priority=5
+	{
+	set_state(c, F);
+
+	if ( c$sip_state$current_response !in c$sip_state$pending &&
+	     (code < 100 && 200 <= code) )
+		++c$sip_state$current_response;
+
+	c$sip$status_code = code;
+	c$sip$status_msg = reason;
+	}
+
+event sip_header(c: connection, is_request: bool, name: string, value: string) &priority=5
+	{
+	if ( ! c?$sip_state )
+		{
+		local s: State;
+		c$sip_state = s;
+		}
+
+	if ( is_request ) # from client
+		{
+		if ( c$sip_state$current_request !in c$sip_state$pending )
+			++c$sip_state$current_request;
+		set_state(c, is_request);
+		switch ( name )
+			{
+			case "CALL-ID":
+				c$sip$call_id = value;
+				break;
+			case "CONTENT-LENGTH", "L":
+				c$sip$request_body_len = to_count(value);
+				break;
+			case "CSEQ":
+				c$sip$seq = value;
+				break;
+			case "DATE":
+				c$sip$date = value;
+				break;
+			case "FROM", "F":
+				c$sip$request_from = split_string1(value, /;[ ]?tag=/)[0];
+				break;
+			case "REPLY-TO":
+				c$sip$reply_to = value;
+				break;
+			case "SUBJECT", "S":
+				c$sip$subject = value;
+				break;
+			case "TO", "T":
+				c$sip$request_to = value;
+				break;
+			case "USER-AGENT":
+				c$sip$user_agent = value;
+				break;
+			case "VIA", "V":
+				c$sip$request_path[|c$sip$request_path|] = split_string1(value, /;[ ]?branch/)[0];
+				break;
+			}
+
+		c$sip_state$pending[c$sip_state$current_request] = c$sip;
+		}
+	else # from server
+		{
+		if ( c$sip_state$current_response !in c$sip_state$pending )
+			++c$sip_state$current_response;
+
+		set_state(c, is_request);
+		switch ( name )
+			{
+			case "CONTENT-LENGTH", "L":
+				c$sip$response_body_len = to_count(value);
+				break;
+			case "CONTENT-TYPE", "C":
+				c$sip$content_type = value;
+				break;
+			case "WARNING":
+				c$sip$warning = value;
+				break;
+			case "FROM", "F":
+				c$sip$response_from = split_string1(value, /;[ ]?tag=/)[0];
+				break;
+			case "TO", "T":
+				c$sip$response_to = value;
+				break;
+			case "VIA", "V":
+				c$sip$response_path[|c$sip$response_path|] = split_string1(value, /;[ ]?branch/)[0];
+				break;
+			}
+
+		c$sip_state$pending[c$sip_state$current_response] = c$sip;
+		}
+	}
+
+event sip_end_entity(c: connection, is_request: bool) &priority = 5
+	{
+	set_state(c, is_request);
+	}
+
+event sip_end_entity(c: connection, is_request: bool) &priority = -5
+	{
+	# The reply body is done so we're ready to log.
+	if ( ! is_request )
+		{
+		Log::write(SIP::LOG, c$sip);
+
+		if ( c$sip$status_code < 100 || 200 <= c$sip$status_code )
+			delete c$sip_state$pending[c$sip_state$current_response];
+
+		if ( ! c$sip?$method || ( c$sip$method == "BYE" &&
+		     c$sip$status_code >= 200 && c$sip$status_code < 300 ) )
+			{
+			flush_pending(c);
+			delete c$sip;
+			delete c$sip_state;
+			}
+		}
+	}
+
+event connection_state_remove(c: connection) &priority=-5
+	{
+	if ( c?$sip_state )
+		{
+		for ( r in c$sip_state$pending )
+			{
+			Log::write(SIP::LOG, c$sip_state$pending[r]);
+			}
+		}
+	}
+

scripts/base/protocols/smtp/main.bro

@@ -29,6 +29,8 @@ export {
 		from:              string          &log &optional;
 		## Contents of the To header.
 		to:                set[string]     &log &optional;
+		## Contents of the CC header.
+		cc:                set[string]     &log &optional;
 		## Contents of the ReplyTo header.
 		reply_to:          string          &log &optional;
 		## Contents of the MsgID header.
@@ -92,13 +94,13 @@ redef likely_server_ports += { ports };
 
 event bro_init() &priority=5
 	{
-	Log::create_stream(SMTP::LOG, [$columns=SMTP::Info, $ev=log_smtp]);
+	Log::create_stream(SMTP::LOG, [$columns=SMTP::Info, $ev=log_smtp, $path="smtp"]);
 	Analyzer::register_for_ports(Analyzer::ANALYZER_SMTP, ports);
 	}
 
 function find_address_in_smtp_header(header: string): string
 {
-	local ips = find_ip_addresses(header);
+	local ips = extract_ip_addresses(header);
 	# If there are more than one IP address found, return the second.
 	if ( |ips| > 1 )
 		return ips[1];
@@ -163,7 +165,7 @@ event smtp_request(c: connection, is_orig: bool, command: string, arg: string) &
 		{
 		if ( ! c$smtp?$rcptto )
 			c$smtp$rcptto = set();
-		add c$smtp$rcptto[split1(arg, /:[[:blank:]]*/)[2]];
+		add c$smtp$rcptto[split_string1(arg, /:[[:blank:]]*/)[1]];
 		c$smtp$has_client_activity = T;
 		}
 
@@ -172,8 +174,8 @@ event smtp_request(c: connection, is_orig: bool, command: string, arg: string) &
 		# Flush last message in case we didn't see the server's acknowledgement.
 		smtp_message(c);
 
-		local partially_done = split1(arg, /:[[:blank:]]*/)[2];
-		c$smtp$mailfrom = split1(partially_done, /[[:blank:]]?/)[1];
+		local partially_done = split_string1(arg, /:[[:blank:]]*/)[1];
+		c$smtp$mailfrom = split_string1(partially_done, /[[:blank:]]?/)[0];
 		c$smtp$has_client_activity = T;
 		}
 	}
@@ -234,14 +236,24 @@ event mime_one_header(c: connection, h: mime_header_rec) &priority=5
 		if ( ! c$smtp?$to )
 			c$smtp$to = set();
 
-		local to_parts = split(h$value, /[[:blank:]]*,[[:blank:]]*/);
+		local to_parts = split_string(h$value, /[[:blank:]]*,[[:blank:]]*/);
 		for ( i in to_parts )
 			add c$smtp$to[to_parts[i]];
 		}
 
+	else if ( h$name == "CC" )
+		{
+		if ( ! c$smtp?$cc )
+			c$smtp$cc = set();
+
+		local cc_parts = split_string(h$value, /[[:blank:]]*,[[:blank:]]*/);
+		for ( i in cc_parts )
+			add c$smtp$cc[cc_parts[i]];
+		}
+
 	else if ( h$name == "X-ORIGINATING-IP" )
 		{
-		local addresses = find_ip_addresses(h$value);
+		local addresses = extract_ip_addresses(h$value);
 		if ( 1 in addresses )
 			c$smtp$x_originating_ip = to_addr(addresses[1]);
 		}

scripts/base/protocols/snmp/main.bro

@@ -66,7 +66,7 @@ redef likely_server_ports += { ports };
 event bro_init() &priority=5
 	{
 	Analyzer::register_for_ports(Analyzer::ANALYZER_SNMP, ports);
-	Log::create_stream(SNMP::LOG, [$columns=SNMP::Info, $ev=log_snmp]);
+	Log::create_stream(SNMP::LOG, [$columns=SNMP::Info, $ev=log_snmp, $path="snmp"]);
 	}
 
 function init_state(c: connection, h: SNMP::Header): Info

scripts/base/protocols/socks/main.bro

@@ -16,8 +16,10 @@ export {
 		id:          conn_id         &log;
 		## Protocol version of SOCKS.
 		version:     count           &log;
-		## Username for the proxy if extracted from the network.
+		## Username used to request a login to the proxy.
 		user:        string          &log &optional;
+		## Password used to request a login to the proxy.
+		password:    string          &log &optional;
 		## Server status for the attempt at using the proxy.
 		status:      string          &log &optional;
 		## Client requested SOCKS address. Could be an address, a name
@@ -41,7 +43,7 @@ redef likely_server_ports += { ports };
 
 event bro_init() &priority=5
 	{
-	Log::create_stream(SOCKS::LOG, [$columns=Info, $ev=log_socks]);
+	Log::create_stream(SOCKS::LOG, [$columns=Info, $ev=log_socks, $path="socks"]);
 	Analyzer::register_for_ports(Analyzer::ANALYZER_SOCKS, ports);
 	}
 
@@ -91,3 +93,21 @@ event socks_reply(c: connection, version: count, reply: count, sa: SOCKS::Addres
 	if ( "SOCKS" in c$service )
 		Log::write(SOCKS::LOG, c$socks);
 	}
+
+event socks_login_userpass_request(c: connection, user: string, password: string) &priority=5
+	{
+	# Authentication only possible with the version 5.
+	set_session(c, 5); 
+
+	c$socks$user = user;
+	c$socks$password = password;
+	}
+
+event socks_login_userpass_reply(c: connection, code: count) &priority=5
+	{
+	# Authentication only possible with the version 5.
+	set_session(c, 5);
+
+	c$socks$status = v5_status[code];
+	}
+

scripts/base/protocols/ssh/README

@@ -1 +1 @@
-Support for Secure Shell (SSH) protocol analysis.
+Support for SSH protocol analysis.

scripts/base/protocols/ssh/__load__.bro

@@ -1,3 +1,2 @@
 @load ./main
-
-@load-sigs ./dpd.sig
+@load-sigs ./dpd.sig

scripts/base/protocols/ssh/dpd.sig

@@ -1,6 +1,6 @@
 signature dpd_ssh_client {
   ip-proto == tcp
-  payload /^[sS][sS][hH]-/
+  payload /^[sS][sS][hH]-[12]\./
   requires-reverse-signature dpd_ssh_server
   enable "ssh"
   tcp-state originator
@@ -8,6 +8,6 @@ signature dpd_ssh_client {
 
 signature dpd_ssh_server {
   ip-proto == tcp
-  payload /^[sS][sS][hH]-/
+  payload /^[sS][sS][hH]-[12]\./
   tcp-state responder
-}
+}

scripts/base/protocols/ssh/main.bro

@@ -1,15 +1,5 @@
-##! Base SSH analysis script.  The heuristic to blindly determine success or
-##! failure for SSH connections is implemented here.  At this time, it only
-##! uses the size of the data being returned from the server to make the
-##! heuristic determination about success of the connection.
-##! Requires that :bro:id:`use_conn_size_analyzer` is set to T!  The heuristic
-##! is not attempted if the connection size analyzer isn't enabled.
+##! Implements base functionality for SSH analysis. Generates the ssh.log file.
 
-@load base/protocols/conn
-@load base/frameworks/notice
-@load base/utils/site
-@load base/utils/thresholds
-@load base/utils/conn-ids
 @load base/utils/directions-and-hosts
 
 module SSH;
@@ -25,45 +15,63 @@ export {
 		uid:             string       &log;
 		## The connection's 4-tuple of endpoint addresses/ports.
 		id:              conn_id      &log;
-		## Indicates if the login was heuristically guessed to be
-		## "success", "failure", or "undetermined".
-		status:          string       &log &default="undetermined";
-		## Direction of the connection.  If the client was a local host
+		## SSH major version (1 or 2)
+		version:         count        &log;
+		## Authentication result (T=success, F=failure, unset=unknown)
+		auth_success:    bool         &log &optional;
+		## Direction of the connection. If the client was a local host
 		## logging into an external host, this would be OUTBOUND. INBOUND
 		## would be set for the opposite situation.
-		# TODO: handle local-local and remote-remote better.
+		# TODO - handle local-local and remote-remote better.
 		direction:       Direction    &log &optional;
-		## Software string from the client.
+		## The client's version string
 		client:          string       &log &optional;
-		## Software string from the server.
+		## The server's version string
 		server:          string       &log &optional;
-		## Indicate if the SSH session is done being watched.
-		done:            bool         &default=F;
+		## The encryption algorithm in use
+		cipher_alg:      string       &log &optional;
+		## The signing (MAC) algorithm in use
+		mac_alg:         string       &log &optional;
+		## The compression algorithm in use
+		compression_alg: string       &log &optional;
+		## The key exchange algorithm in use
+		kex_alg:         string       &log &optional;
+		## The server host key's algorithm
+		host_key_alg:    string       &log &optional;
+		## The server's key fingerprint
+		host_key:        string       &log &optional;
 	};
 
-	## The size in bytes of data sent by the server at which the SSH
-	## connection is presumed to be successful.
-	const authentication_data_size = 4000 &redef;
+	## The set of compression algorithms. We can't accurately determine
+	## authentication success or failure when compression is enabled.
+	const compression_algorithms = set("zlib", "zlib@openssh.com") &redef;
 
 	## If true, we tell the event engine to not look at further data
 	## packets after the initial SSH handshake. Helps with performance
 	## (especially with large file transfers) but precludes some
-	## kinds of analyses.
-	const skip_processing_after_detection = F &redef;
+	## kinds of analyses. Defaults to T.
+	const skip_processing_after_detection = T &redef;
 
-	## Event that is generated when the heuristic thinks that a login
-	## was successful.
-	global heuristic_successful_login: event(c: connection);
-
-	## Event that is generated when the heuristic thinks that a login
-	## failed.
-	global heuristic_failed_login: event(c: connection);
-
-	## Event that can be handled to access the :bro:type:`SSH::Info`
-	## record as it is sent on to the logging framework.
+	## Event that can be handled to access the SSH record as it is sent on
+	## to the logging framework.
 	global log_ssh: event(rec: Info);
+
+	## Event that can be handled when the analyzer sees an SSH server host
+	## key. This abstracts :bro:id:`ssh1_server_host_key` and
+	## :bro:id:`ssh2_server_host_key`.
+	global ssh_server_host_key: event(c: connection, hash: string);
 }
 
+redef record Info += {
+	# This connection has been logged (internal use)
+	logged:       bool         &default=F;
+	# Number of failures seen (internal use)
+	num_failures: count        &default=0;
+	# Store capabilities from the first host for
+	# comparison with the second (internal use)
+	capabilities: Capabilities &optional;
+};
+
 redef record connection += {
 	ssh: Info &optional;
 };
@@ -72,133 +80,156 @@ const ports = { 22/tcp };
 redef likely_server_ports += { ports };
 
 event bro_init() &priority=5
-{
-	Log::create_stream(SSH::LOG, [$columns=Info, $ev=log_ssh]);
+	{
 	Analyzer::register_for_ports(Analyzer::ANALYZER_SSH, ports);
-}
+	Log::create_stream(SSH::LOG, [$columns=Info, $ev=log_ssh, $path="ssh"]);
+	}
 
 function set_session(c: connection)
 	{
 	if ( ! c?$ssh )
 		{
-		local info: Info;
-		info$ts=network_time();
-		info$uid=c$uid;
-		info$id=c$id;
+		local info: SSH::Info;
+		info$ts  = network_time();
+		info$uid = c$uid;
+		info$id  = c$id;
+
+		# If both hosts are local or non-local, we can't reliably set a direction.
+		if ( Site::is_local_addr(c$id$orig_h) != Site::is_local_addr(c$id$resp_h) )
+			info$direction = Site::is_local_addr(c$id$orig_h) ? OUTBOUND: INBOUND;
 		c$ssh = info;
 		}
 	}
 
-function check_ssh_connection(c: connection, done: bool)
-	{
-	# If already done watching this connection, just return.
-	if ( c$ssh$done )
-		return;
-
-	if ( done )
-		{
-		# If this connection is done, then we can look to see if
-		# this matches the conditions for a failed login.  Failed
-		# logins are only detected at connection state removal.
-
-		if ( # Require originators and responders to have sent at least 50 bytes.
-		     c$orig$size > 50 && c$resp$size > 50 &&
-		     # Responders must be below 4000 bytes.
-		     c$resp$size < authentication_data_size &&
-		     # Responder must have sent fewer than 40 packets.
-		     c$resp$num_pkts < 40 &&
-		     # If there was a content gap we can't reliably do this heuristic.
-		     c?$conn && c$conn$missed_bytes == 0 )# &&
-		     # Only "normal" connections can count.
-		     #c$conn?$conn_state && c$conn$conn_state in valid_states )
-			{
-			c$ssh$status = "failure";
-			event SSH::heuristic_failed_login(c);
-			}
-
-		if ( c$resp$size >= authentication_data_size )
-			{
-			c$ssh$status = "success";
-			event SSH::heuristic_successful_login(c);
-			}
-		}
-	else
-		{
-		# If this connection is still being tracked, then it's possible
-		# to watch for it to be a successful connection.
-		if ( c$resp$size >= authentication_data_size )
-			{
-			c$ssh$status = "success";
-			event SSH::heuristic_successful_login(c);
-			}
-		else
-			# This connection must be tracked longer.  Let the scheduled
-			# check happen again.
-			return;
-		}
-
-	# Set the direction for the log.
-	c$ssh$direction = Site::is_local_addr(c$id$orig_h) ? OUTBOUND : INBOUND;
-
-	# Set the "done" flag to prevent the watching event from rescheduling
-	# after detection is done.
-	c$ssh$done=T;
-
-	if ( skip_processing_after_detection )
-		{
-		# Stop watching this connection, we don't care about it anymore.
-		skip_further_processing(c$id);
-		set_record_packets(c$id, F);
-		}
-	}
-
-
-event heuristic_successful_login(c: connection) &priority=-5
-	{
-	Log::write(SSH::LOG, c$ssh);
-	}
-
-event heuristic_failed_login(c: connection) &priority=-5
-	{
-	Log::write(SSH::LOG, c$ssh);
-	}
-
-event connection_state_remove(c: connection) &priority=-5
-	{
-	if ( c?$ssh )
-		{
-		check_ssh_connection(c, T);
-		if ( c$ssh$status == "undetermined" )
-			Log::write(SSH::LOG, c$ssh);
-		}
-	}
-
-event ssh_watcher(c: connection)
-	{
-	local id = c$id;
-	# don't go any further if this connection is gone already!
-	if ( ! connection_exists(id) )
-		return;
-
-	lookup_connection(c$id);
-	check_ssh_connection(c, F);
-	if ( ! c$ssh$done )
-		schedule +15secs { ssh_watcher(c) };
-	}
-
-event ssh_server_version(c: connection, version: string) &priority=5
+event ssh_server_version(c: connection, version: string)
 	{
 	set_session(c);
 	c$ssh$server = version;
 	}
 
-event ssh_client_version(c: connection, version: string) &priority=5
+event ssh_client_version(c: connection, version: string)
 	{
 	set_session(c);
 	c$ssh$client = version;
 
-	# The heuristic detection for SSH relies on the ConnSize analyzer.
-	# Don't do the heuristics if it's disabled.
-	if ( use_conn_size_analyzer )
-		schedule +15secs { ssh_watcher(c) };
+	if ( ( |version| > 3 ) && ( version[4] == "1" ) )
+		c$ssh$version = 1;
+	if ( ( |version| > 3 ) && ( version[4] == "2" ) )
+		c$ssh$version = 2;
+	}
+
+event ssh_auth_successful(c: connection, auth_method_none: bool) &priority=5
+	{
+	# TODO - what to do here?
+	if ( !c?$ssh || ( c$ssh?$auth_success && c$ssh$auth_success ) )
+		return;
+
+	# We can't accurately tell for compressed streams
+	if ( c$ssh?$compression_alg && ( c$ssh$compression_alg in compression_algorithms ) )
+		return;
+
+	c$ssh$auth_success = T;
+
+	if ( skip_processing_after_detection)
+		{
+		skip_further_processing(c$id);
+		set_record_packets(c$id, F);
+		}
+	}
+
+event ssh_auth_successful(c: connection, auth_method_none: bool) &priority=-5
+	{
+	if ( c?$ssh && !c$ssh$logged )
+		{
+		c$ssh$logged = T;
+		Log::write(SSH::LOG, c$ssh);
+		}
+	}
+
+event ssh_auth_failed(c: connection) &priority=5
+	{
+	if ( !c?$ssh || ( c$ssh?$auth_success && !c$ssh$auth_success ) )
+		return;
+
+	# We can't accurately tell for compressed streams
+	if ( c$ssh?$compression_alg && ( c$ssh$compression_alg in compression_algorithms ) )
+		return;
+
+	c$ssh$auth_success = F;
+	c$ssh$num_failures += 1;
+	}
+
+# Determine the negotiated algorithm
+function find_alg(client_algorithms: vector of string, server_algorithms: vector of string): string
+	{
+	for ( i in client_algorithms )
+		for ( j in server_algorithms )
+			if ( client_algorithms[i] == server_algorithms[j] )
+				return client_algorithms[i];
+	return "Algorithm negotiation failed";
+	}
+
+# This is a simple wrapper around find_alg for cases where client to server and server to client
+# negotiate different algorithms. This is rare, but provided for completeness.
+function find_bidirectional_alg(client_prefs: Algorithm_Prefs, server_prefs: Algorithm_Prefs): string
+	{
+	local c_to_s = find_alg(client_prefs$client_to_server, server_prefs$client_to_server);
+	local s_to_c = find_alg(client_prefs$server_to_client, server_prefs$server_to_client);
+
+	# Usually these are the same, but if they're not, return the details
+	return c_to_s == s_to_c ? c_to_s : fmt("To server: %s, to client: %s", c_to_s, s_to_c);
+	}
+	
+event ssh_capabilities(c: connection, cookie: string, capabilities: Capabilities)
+	{
+	if ( !c?$ssh || ( c$ssh?$capabilities && c$ssh$capabilities$is_server == capabilities$is_server ) )
+		return;
+
+	if ( !c$ssh?$capabilities )
+		{
+		c$ssh$capabilities = capabilities;
+		return;
+		}
+
+	local client_caps = capabilities$is_server ? c$ssh$capabilities : capabilities;
+	local server_caps = capabilities$is_server ? capabilities : c$ssh$capabilities;
+
+	c$ssh$cipher_alg      = find_bidirectional_alg(client_caps$encryption_algorithms,
+	                                               server_caps$encryption_algorithms);
+	c$ssh$mac_alg         = find_bidirectional_alg(client_caps$mac_algorithms,
+	                                               server_caps$mac_algorithms);
+	c$ssh$compression_alg = find_bidirectional_alg(client_caps$compression_algorithms,
+	                                               server_caps$compression_algorithms);
+	c$ssh$kex_alg         = find_alg(client_caps$kex_algorithms, server_caps$kex_algorithms);	
+	c$ssh$host_key_alg    = find_alg(client_caps$server_host_key_algorithms,
+	                                 server_caps$server_host_key_algorithms);
+	}
+
+event connection_state_remove(c: connection) &priority=-5
+	{
+	if ( c?$ssh && !c$ssh$logged && c$ssh?$client && c$ssh?$server )
+		{
+		c$ssh$logged = T;
+		Log::write(SSH::LOG, c$ssh);
+		}
+	}
+
+function generate_fingerprint(c: connection, key: string)
+	{
+	if ( !c?$ssh )
+		return;
+
+	local lx = str_split(md5_hash(key), vector(2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30));
+	lx[0] = "";
+	c$ssh$host_key = sub(join_string_vec(lx, ":"), /:/, "");
+	}
+
+event ssh1_server_host_key(c: connection, p: string, e: string) &priority=5
+	{
+	generate_fingerprint(c, e + p);
+	}
+
+event ssh2_server_host_key(c: connection, key: string) &priority=5
+	{
+	generate_fingerprint(c, key);
 	}

scripts/base/protocols/ssl/consts.bro

@@ -6,6 +6,11 @@ export {
 	const TLSv10 = 0x0301;
 	const TLSv11 = 0x0302;
 	const TLSv12 = 0x0303;
+
+	const DTLSv10 = 0xFEFF;
+	# DTLSv11 does not exist
+	const DTLSv12 = 0xFEFD;
+
 	## Mapping between the constants and string values for SSL/TLS versions.
 	const version_strings: table[count] of string = {
 		[SSLv2] = "SSLv2",
@@ -13,6 +18,8 @@ export {
 		[TLSv10] = "TLSv10",
 		[TLSv11] = "TLSv11",
 		[TLSv12] = "TLSv12",
+		[DTLSv10] = "DTLSv10",
+		[DTLSv12] = "DTLSv12"
 	} &default=function(i: count):string { return fmt("unknown-%d", i); };
 
 	## TLS content types:
@@ -30,6 +37,7 @@ export {
 	const HELLO_REQUEST       = 0;
 	const CLIENT_HELLO        = 1;
 	const SERVER_HELLO        = 2;
+	const HELLO_VERIFY_REQUEST = 3; # RFC 6347
 	const SESSION_TICKET      = 4; # RFC 5077
 	const CERTIFICATE         = 11;
 	const SERVER_KEY_EXCHANGE = 12;
@@ -40,6 +48,7 @@ export {
 	const FINISHED            = 20;
 	const CERTIFICATE_URL     = 21; # RFC 3546
 	const CERTIFICATE_STATUS  = 22; # RFC 3546
+	const SUPPLEMENTAL_DATA   = 23; # RFC 4680
 
 	## Mapping between numeric codes and human readable strings for alert
 	## levels.
@@ -111,8 +120,9 @@ export {
 		[18] = "signed_certificate_timestamp",
 		[19] = "client_certificate_type",
 		[20] = "server_certificate_type",
-		[21] = "padding", # temporary till 2015-03-12
-		[22] = "encrypt_then_mac", # temporary till 2015-06-05
+		[21] = "padding", # temporary till 2016-03-12
+		[22] = "encrypt_then_mac",
+		[23] = "extended_master_secret",
 		[35] = "SessionTicket TLS",
 		[40] = "extended_random",
 		[13172] = "next_protocol_negotiation",
@@ -155,6 +165,12 @@ export {
 		[26] = "brainpoolP256r1",
 		[27] = "brainpoolP384r1",
 		[28] = "brainpoolP512r1",
+		# draft-ietf-tls-negotiated-ff-dhe-05
+		[256] = "ffdhe2048",
+		[257] = "ffdhe3072",
+		[258] = "ffdhe4096",
+		[259] = "ffdhe6144",
+		[260] = "ffdhe8192",
 		[0xFF01] = "arbitrary_explicit_prime_curves",
 		[0xFF02] = "arbitrary_explicit_char2_curves"
 	} &default=function(i: count):string { return fmt("unknown-%d", i); };

scripts/base/protocols/ssl/dpd.sig

@@ -1,7 +1,7 @@
 signature dpd_ssl_server {
   ip-proto == tcp
   # Server hello.
-  payload /^(\x16\x03[\x00\x01\x02\x03]..\x02...\x03[\x00\x01\x02\x03]|...?\x04..\x00\x02).*/
+  payload /^((\x15\x03[\x00\x01\x02\x03]....)?\x16\x03[\x00\x01\x02\x03]..\x02...\x03[\x00\x01\x02\x03]|...?\x04..\x00\x02).*/
   requires-reverse-signature dpd_ssl_client
   enable "ssl"
   tcp-state responder
@@ -13,3 +13,10 @@ signature dpd_ssl_client {
   payload /^(\x16\x03[\x00\x01\x02\x03]..\x01...\x03[\x00\x01\x02\x03]|...?\x01[\x00\x03][\x00\x01\x02\x03]).*/
   tcp-state originator
 }
+
+signature dpd_dtls_client {
+  ip-proto == udp
+	# Client hello.
+	payload /^\x16\xfe[\xff\xfd]\x00\x00\x00\x00\x00\x00\x00...\x01...........\xfe[\xff\xfd].*/
+	enable "dtls"
+}

scripts/base/protocols/ssl/files.bro

@@ -85,6 +85,10 @@ event bro_init() &priority=5
 	Files::register_protocol(Analyzer::ANALYZER_SSL,
 	                         [$get_file_handle = SSL::get_file_handle,
 	                          $describe        = SSL::describe_file]);
+
+	Files::register_protocol(Analyzer::ANALYZER_DTLS,
+	                         [$get_file_handle = SSL::get_file_handle,
+	                          $describe        = SSL::describe_file]);
 	}
 
 event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priority=5

scripts/base/protocols/ssl/main.bro

@@ -12,7 +12,7 @@ export {
 		## Time when the SSL connection was first detected.
 		ts:               time             &log;
 		## Unique ID for the connection.
-		uid:         string          &log;
+		uid:              string           &log;
 		## The connection's 4-tuple of endpoint addresses/ports.
 		id:               conn_id          &log;
 		## SSL/TLS version that the server offered.
@@ -25,9 +25,25 @@ export {
 		## indicates the server name that the client was requesting.
 		server_name:      string           &log &optional;
 		## Session ID offered by the client for session resumption.
-		session_id:       string           &log &optional;
+		## Not used for logging.
+		session_id:       string           &optional;
+		## Flag to indicate if the session was resumed reusing
+		## the key material exchanged in an earlier connection.
+		resumed:          bool             &log &default=F;
+		## Flag to indicate if we saw a non-empty session ticket being
+		## sent by the client using an empty session ID. This value
+		## is used to determine if a session is being resumed. It's
+		## not logged.
+		client_ticket_empty_session_seen: bool &default=F;
+		## Flag to indicate if we saw a client key exchange message sent
+		## by the client. This value is used to determine if a session
+		## is being resumed. It's not logged.
+		client_key_exchange_seen: bool     &default=F;
 		## Last alert that was seen during the connection.
 		last_alert:       string           &log &optional;
+		## Next protocol the server chose using the application layer
+		## next protocol extension, if present.
+		next_protocol:    string           &log &optional;
 
 		## The analyzer ID used for the analyzer instance attached
 		## to each connection.  It is not used for logging since it's a
@@ -36,11 +52,11 @@ export {
 
 		## Flag to indicate if this ssl session has been established
 		## succesfully, or if it was aborted during the handshake.
-		established:  bool &log &default=F;
+		established:      bool             &log &default=F;
 
 		## Flag to indicate if this record already has been logged, to
 		## prevent duplicates.
-		logged:  bool  &default=F;
+		logged:           bool             &default=F;
 	};
 
 	## The default root CA bundle.  By default, the mozilla-ca-list.bro
@@ -76,16 +92,22 @@ redef record Info += {
 		delay_tokens: set[string] &optional;
 };
 
-const ports = {
+const ssl_ports = {
 	443/tcp, 563/tcp, 585/tcp, 614/tcp, 636/tcp,
 	989/tcp, 990/tcp, 992/tcp, 993/tcp, 995/tcp, 5223/tcp
 };
-redef likely_server_ports += { ports };
+
+# There are no well known DTLS ports at the moment. Let's
+# just add 443 for now for good measure - who knows :)
+const dtls_ports = { 443/udp };
+
+redef likely_server_ports += { ssl_ports, dtls_ports };
 
 event bro_init() &priority=5
 	{
-	Log::create_stream(SSL::LOG, [$columns=Info, $ev=log_ssl]);
-	Analyzer::register_for_ports(Analyzer::ANALYZER_SSL, ports);
+	Log::create_stream(SSL::LOG, [$columns=Info, $ev=log_ssl, $path="ssl"]);
+	Analyzer::register_for_ports(Analyzer::ANALYZER_SSL, ssl_ports);
+	Analyzer::register_for_ports(Analyzer::ANALYZER_DTLS, dtls_ports);
 	}
 
 function set_session(c: connection)
@@ -149,8 +171,11 @@ event ssl_client_hello(c: connection, version: count, possible_ts: time, client_
 	set_session(c);
 
 	# Save the session_id if there is one set.
-	if ( session_id != /^\x00{32}$/ )
+	if ( |session_id| > 0 && session_id != /^\x00{32}$/ )
+		{
 		c$ssl$session_id = bytestring_to_hexstr(session_id);
+		c$ssl$client_ticket_empty_session_seen = F;
+		}
 	}
 
 event ssl_server_hello(c: connection, version: count, possible_ts: time, server_random: string, session_id: string, cipher: count, comp_method: count) &priority=5
@@ -159,6 +184,9 @@ event ssl_server_hello(c: connection, version: count, possible_ts: time, server_
 
 	c$ssl$version = version_strings[version];
 	c$ssl$cipher = cipher_desc[cipher];
+
+	if ( c$ssl?$session_id && c$ssl$session_id == bytestring_to_hexstr(session_id) )
+		c$ssl$resumed = T;
 	}
 
 event ssl_server_curve(c: connection, curve: count) &priority=5
@@ -180,6 +208,45 @@ event ssl_extension_server_name(c: connection, is_orig: bool, names: string_vec)
 		}
 	}
 
+event ssl_extension_application_layer_protocol_negotiation(c: connection, is_orig: bool, protocols: string_vec)
+	{
+	set_session(c);
+
+	if ( is_orig )
+		return;
+
+	if ( |protocols| > 0 )
+		c$ssl$next_protocol = protocols[0];
+	}
+
+event ssl_handshake_message(c: connection, is_orig: bool, msg_type: count, length: count) &priority=5
+	{
+	set_session(c);
+
+	if ( is_orig && msg_type == SSL::CLIENT_KEY_EXCHANGE )
+		c$ssl$client_key_exchange_seen = T;
+	}
+
+# Extension event is fired _before_ the respective client or server hello.
+# Important for client_ticket_empty_session_seen.
+event ssl_extension(c: connection, is_orig: bool, code: count, val: string) &priority=5
+	{
+	set_session(c);
+
+	if ( is_orig && SSL::extensions[code] == "SessionTicket TLS" && |val| > 0 )
+		# In this case, we might have an empty ID. Set back to F in client_hello event
+		# if it is not empty after all.
+		c$ssl$client_ticket_empty_session_seen = T;
+	}
+
+event ssl_change_cipher_spec(c: connection, is_orig: bool) &priority=5
+	{
+	set_session(c);
+
+	if ( is_orig && c$ssl$client_ticket_empty_session_seen && ! c$ssl$client_key_exchange_seen )
+		c$ssl$resumed = T;
+	}
+
 event ssl_alert(c: connection, is_orig: bool, level: count, desc: count) &priority=5
 	{
 	set_session(c);
@@ -207,7 +274,7 @@ event connection_state_remove(c: connection) &priority=-5
 
 event protocol_confirmation(c: connection, atype: Analyzer::Tag, aid: count) &priority=5
 	{
-	if ( atype == Analyzer::ANALYZER_SSL )
+	if ( atype == Analyzer::ANALYZER_SSL || atype == Analyzer::ANALYZER_DTLS )
 		{
 		set_session(c);
 		c$ssl$analyzer_id = aid;
@@ -217,6 +284,6 @@ event protocol_confirmation(c: connection, atype: Analyzer::Tag, aid: count) &pr
 event protocol_violation(c: connection, atype: Analyzer::Tag, aid: count,
                          reason: string) &priority=5
 	{
-	if ( c?$ssl )
+	if ( c?$ssl && ( atype == Analyzer::ANALYZER_SSL || atype == Analyzer::ANALYZER_DTLS ) )
 		finish(c, T);
 	}

scripts/base/protocols/syslog/main.bro

@@ -35,7 +35,7 @@ redef likely_server_ports += { ports };
 
 event bro_init() &priority=5
 	{
-	Log::create_stream(Syslog::LOG, [$columns=Info]);
+	Log::create_stream(Syslog::LOG, [$columns=Info, $path="syslog"]);
 	Analyzer::register_for_ports(Analyzer::ANALYZER_SYSLOG, ports);
 	}
 

scripts/base/protocols/tunnels/dpd.sig

@@ -9,6 +9,6 @@ signature dpd_ayiya {
 
 signature dpd_teredo {
   ip-proto = udp
-  payload /^(\x00\x00)|(\x00\x01)|([\x60-\x6f])/
+  payload /^(\x00\x00)|(\x00\x01)|([\x60-\x6f].{7}((\x20\x01\x00\x00)).{28})|([\x60-\x6f].{23}((\x20\x01\x00\x00))).{12}/
   enable "teredo"
 }

scripts/base/utils/active-http.bro

@@ -65,12 +65,14 @@ function request2curl(r: Request, bodyfile: string, headersfile: string): string
 	cmd = fmt("%s -m %.0f", cmd, r$max_time);
 
 	if ( r?$client_data )
-		cmd = fmt("%s -d -", cmd);
+		cmd = fmt("%s -d @-", cmd);
 
 	if ( r?$addl_curl_args )
 		cmd = fmt("%s %s", cmd, r$addl_curl_args);
 
 	cmd = fmt("%s \"%s\"", cmd, str_shell_escape(r$url));
+	# Make sure file will exist even if curl did not write one.
+	cmd = fmt("%s && touch %s", cmd, str_shell_escape(bodyfile));
 	return cmd;
 	}
 
@@ -103,21 +105,21 @@ function request(req: Request): ActiveHTTP::Response
 			# The reply is the first line.
 			if ( i == 0 )
 				{
-				local response_line = split_n(headers[0], /[[:blank:]]+/, F, 2);
+				local response_line = split_string_n(headers[0], /[[:blank:]]+/, F, 2);
 				if ( |response_line| != 3 )
 					return resp;
 
-				resp$code = to_count(response_line[2]);
-				resp$msg = response_line[3];
+				resp$code = to_count(response_line[1]);
+				resp$msg = response_line[2];
 				resp$body = join_string_vec(result$files[bodyfile], "");
 				}
 			else
 				{
 				local line = headers[i];
-				local h = split1(line, /:/);
+				local h = split_string1(line, /:/);
 				if ( |h| != 2 )
 					next;
-				resp$headers[h[1]] = sub_bytes(h[2], 0, |h[2]|-1);
+				resp$headers[h[0]] = sub_bytes(h[1], 0, |h[1]|-1);
 				}
 			}
 		return resp;

scripts/base/utils/addrs.bro

@@ -32,7 +32,7 @@ const ip_addr_regex =
 ## octets: an array of strings to check for valid octet values.
 ##
 ## Returns: T if every element is between 0 and 255, inclusive, else F.
-function has_valid_octets(octets: string_array): bool
+function has_valid_octets(octets: string_vec): bool
 	{
 	local num = 0;
 	for ( i in octets )
@@ -51,10 +51,10 @@ function has_valid_octets(octets: string_array): bool
 ## Returns: T if the string is a valid IPv4 or IPv6 address format.
 function is_valid_ip(ip_str: string): bool
 	{
-	local octets: string_array;
+	local octets: string_vec;
 	if ( ip_str == ipv4_addr_regex )
 		{
-		octets = split(ip_str, /\./);
+		octets = split_string(ip_str, /\./);
 		if ( |octets| != 4 )
 			return F;
 
@@ -67,13 +67,13 @@ function is_valid_ip(ip_str: string): bool
 			{
 			# the regexes for hybrid IPv6-IPv4 address formats don't for valid
 			# octets within the IPv4 part, so do that now
-			octets = split(ip_str, /\./);
+			octets = split_string(ip_str, /\./);
 			if ( |octets| != 4 )
 				return F;
 
 			# get rid of remaining IPv6 stuff in first octet
-			local tmp = split(octets[1], /:/);
-			octets[1] = tmp[|tmp|];
+			local tmp = split_string(octets[0], /:/);
+			octets[0] = tmp[|tmp| - 1];
 
 			return has_valid_octets(octets);
 			}
@@ -92,14 +92,32 @@ function is_valid_ip(ip_str: string): bool
 ## input: a string that may contain an IP address anywhere within it.
 ##
 ## Returns: an array containing all valid IP address strings found in *input*.
-function find_ip_addresses(input: string): string_array
+function find_ip_addresses(input: string): string_array &deprecated
 	{
-	local parts = split_all(input, ip_addr_regex);
+	local parts = split_string_all(input, ip_addr_regex);
 	local output: string_array;
 
 	for ( i in parts )
 		{
-		if ( i % 2 == 0 && is_valid_ip(parts[i]) )
+		if ( i % 2 == 1 && is_valid_ip(parts[i]) )
+			output[|output|] = parts[i];
+		}
+	return output;
+	}
+
+## Extracts all IP (v4 or v6) address strings from a given string.
+##
+## input: a string that may contain an IP address anywhere within it.
+##
+## Returns: an array containing all valid IP address strings found in *input*.
+function extract_ip_addresses(input: string): string_vec
+	{
+	local parts = split_string_all(input, ip_addr_regex);
+	local output: string_vec;
+
+	for ( i in parts )
+		{
+		if ( i % 2 == 1 && is_valid_ip(parts[i]) )
 			output[|output|] = parts[i];
 		}
 	return output;

scripts/base/utils/exec.bro

@@ -82,9 +82,9 @@ event Exec::line(description: Input::EventDescription, tpe: Input::Event, s: str
 
 event Exec::file_line(description: Input::EventDescription, tpe: Input::Event, s: string)
 	{
-	local parts = split1(description$name, /_/);
-	local name = parts[1];
-	local track_file = parts[2];
+	local parts = split_string1(description$name, /_/);
+	local name = parts[0];
+	local track_file = parts[1];
 
 	local result = results[name];
 	if ( ! result?$files )
@@ -96,15 +96,25 @@ event Exec::file_line(description: Input::EventDescription, tpe: Input::Event, s
 		result$files[track_file][|result$files[track_file]|] = s;
 	}
 
-event Input::end_of_data(name: string, source:string)
+event Input::end_of_data(orig_name: string, source:string)
 	{
-	local parts = split1(name, /_/);
-	name = parts[1];
+	local name = orig_name;
+	local parts = split_string1(name, /_/);
+	name = parts[0];
 
 	if ( name !in pending_commands || |parts| < 2 )
 		return;
 
-	local track_file = parts[2];
+	local track_file = parts[1];
+
+	# If the file is empty, still add it to the result$files table. This is needed
+	# because it is expected that the file was read even if it was empty.
+	local result = results[name];
+	if ( ! result?$files )
+		result$files = table();
+
+	if ( track_file !in result$files )
+		result$files[track_file] = vector();
 
 	Input::remove(name);
 

scripts/base/utils/files.bro

@@ -23,7 +23,7 @@ function extract_filename_from_content_disposition(data: string): string
 
 	# Remove quotes around the filename if they are there.
 	if ( /^\"/ in filename )
-		filename = split_n(filename, /\"/, F, 2)[2];
+		filename = split_string_n(filename, /\"/, F, 2)[1];
 
 	# Remove the language and encoding if it's there.
 	if ( /^[a-zA-Z0-9\!#$%&+-^_`{}~]+'[a-zA-Z0-9\!#$%&+-^_`{}~]*'/ in filename )

scripts/base/utils/numbers.bro

@@ -2,9 +2,9 @@
 ## If no integer can be found, 0 is returned.
 function extract_count(s: string): count
 	{
-	local parts = split_n(s, /[0-9]+/, T, 1);
-	if ( 2 in parts )
-		return to_count(parts[2]);
+	local parts = split_string_n(s, /[0-9]+/, T, 1);
+	if ( 1 in parts )
+		return to_count(parts[1]);
 	else
 		return 0;
-	}
+	}

scripts/base/utils/paths.bro

@@ -13,12 +13,12 @@ const absolute_path_pat = /(\/|[A-Za-z]:[\\\/]).*/;
 function extract_path(input: string): string
 	{
 	const dir_pattern = /(\/|[A-Za-z]:[\\\/])([^\"\ ]|(\\\ ))*/;
-	local parts = split_all(input, dir_pattern);
+	local parts = split_string_all(input, dir_pattern);
 
 	if ( |parts| < 3 )
 		return "";
 
-	return parts[2];
+	return parts[1];
 	}
 
 ## Compresses a given path by removing '..'s and the parent directory it
@@ -31,27 +31,27 @@ function compress_path(dir: string): string
 	{
 	const cdup_sep = /((\/)*([^\/]|\\\/)+)?((\/)+\.\.(\/)*)/;
 
-	local parts = split_n(dir, cdup_sep, T, 1);
+	local parts = split_string_n(dir, cdup_sep, T, 1);
 	if ( |parts| > 1 )
 		{
 		# reaching a point with two parent dir references back-to-back means
 		# we don't know about anything higher in the tree to pop off
-		if ( parts[2] == "../.." )
-			return cat_string_array(parts);
-		if ( sub_bytes(parts[2], 0, 1) == "/" )
-			parts[2] = "/";
+		if ( parts[1] == "../.." )
+			return join_string_vec(parts, "");
+		if ( sub_bytes(parts[1], 0, 1) == "/" )
+			parts[1] = "/";
 		else
-			parts[2] = "";
-		dir = cat_string_array(parts);
+			parts[1] = "";
+		dir = join_string_vec(parts, "");
 		return compress_path(dir);
 		}
 
 	const multislash_sep = /(\/\.?){2,}/;
-	parts = split_all(dir, multislash_sep);
+	parts = split_string_all(dir, multislash_sep);
 	for ( i in parts )
-		if ( i % 2 == 0 )
+		if ( i % 2 == 1 )
 			parts[i] = "/";
-	dir = cat_string_array(parts);
+	dir = join_string_vec(parts, "");
 
 	# remove trailing slashes from path
 	if ( |dir| > 1 && sub_bytes(dir, |dir|, 1) == "/" )

scripts/base/utils/patterns.bro

@@ -50,11 +50,11 @@ type PatternMatchResult: record {
 ## Returns: a record indicating the match status.
 function match_pattern(s: string, p: pattern): PatternMatchResult
 	{
-	local a = split_n(s, p, T, 1);
+	local a = split_string_n(s, p, T, 1);
 
 	if ( |a| == 1 )
 		# no match
 		return [$matched = F, $str = "", $off = 0];
 	else
-		return [$matched = T, $str = a[2], $off = |a[1]| + 1];
+		return [$matched = T, $str = a[1], $off = |a[0]| + 1];
 	}