From f07fdc255f8dc00fe5087b9c35c3e42d0442fd42 Mon Sep 17 00:00:00 2001 From: Andrew Woodford Date: Thu, 21 Dec 2017 16:34:26 +0000 Subject: [PATCH 01/56] SSH protocol now assesses the packet length at an earlier stage within binpac. Stops SSH analyzer constantly raising binpac exceptions. Seems to be because a packet continues to go through binpac when empty and only calls the next packet when asked for more data and not on operations. --- src/analyzer/protocol/ssh/ssh-analyzer.pac | 12 +++++++++++ src/analyzer/protocol/ssh/ssh-protocol.pac | 25 +++++++++++----------- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/src/analyzer/protocol/ssh/ssh-analyzer.pac b/src/analyzer/protocol/ssh/ssh-analyzer.pac index 598dc869ab..bdb2553849 100644 --- a/src/analyzer/protocol/ssh/ssh-analyzer.pac +++ b/src/analyzer/protocol/ssh/ssh-analyzer.pac @@ -173,6 +173,18 @@ refine flow SSH_Flow += { connection()->bro_analyzer()->ProtocolConfirmation(); return true; %} + + function get_kex_length(v: int, packet_length: uint32): int + %{ + switch (v) { + case SSH1: + return packet_length + 4 + 8 -(packet_length%8); + case SSH2: + return packet_length + 4; + default: + return 1; //currently causes the rest of the packet to dump + } + %} }; refine typeattr SSH_Version += &let { diff --git a/src/analyzer/protocol/ssh/ssh-protocol.pac b/src/analyzer/protocol/ssh/ssh-protocol.pac index 3b147f6b6e..a86e6cef8e 100644 --- a/src/analyzer/protocol/ssh/ssh-protocol.pac +++ b/src/analyzer/protocol/ssh/ssh-protocol.pac @@ -22,21 +22,23 @@ type SSH_Version(is_orig: bool) = record { update_version : bool = $context.connection.update_version(version, is_orig); }; -type SSH_Key_Exchange(is_orig: bool) = case $context.connection.get_version() of { - SSH1 -> ssh1_msg : SSH1_Key_Exchange(is_orig); - SSH2 -> ssh2_msg : SSH2_Key_Exchange(is_orig); -}; +type SSH_Key_Exchange(is_orig: bool) = record { + packet_length: uint32; + key_ex: case $context.connection.get_version() of { + SSH1 -> ssh1_msg : SSH1_Key_Exchange(is_orig, packet_length); + SSH2 -> ssh2_msg : SSH2_Key_Exchange(is_orig, packet_length); + }; +} &length = $context.flow.get_kex_length($context.connection.get_version(), packet_length); # SSH1 constructs ################# -type SSH1_Key_Exchange(is_orig: bool) = record { - packet_length : uint32; +type SSH1_Key_Exchange(is_orig: bool, packet_length: uint32) = record { pad_fill : bytestring &length = 8 - (packet_length % 8); msg_type : uint8; message : SSH1_Message(is_orig, msg_type, packet_length - 5); crc : uint32; -} &length = packet_length + 4 + 8 - (packet_length % 8); +} &length = packet_length + 8 - (packet_length % 8); type SSH1_Message(is_orig: bool, msg_type: uint8, length: uint32) = case msg_type of { SSH_SMSG_PUBLIC_KEY -> public_key : SSH1_PUBLIC_KEY(length); @@ -73,8 +75,7 @@ type ssh1_mp_int = record { ## SSH2 -type SSH2_Header(is_orig: bool) = record { - packet_length : uint32; +type SSH2_Header(is_orig: bool, packet_length: uint32) = record { padding_length : uint8; msg_type : uint8; } &let { @@ -82,11 +83,11 @@ type SSH2_Header(is_orig: bool) = record { detach : bool = $context.connection.update_state(ENCRYPTED, is_orig) &if(msg_type == MSG_NEWKEYS); }; -type SSH2_Key_Exchange(is_orig: bool) = record { - header : SSH2_Header(is_orig); +type SSH2_Key_Exchange(is_orig: bool, packet_length: uint32) = record { + header : SSH2_Header(is_orig, packet_length); payload : SSH2_Message(is_orig, header.msg_type, header.payload_length); pad : bytestring &length=header.padding_length; -} &length=header.packet_length + 4; +} &length=packet_length; type SSH2_Message(is_orig: bool, msg_type: uint8, length: uint32) = case $context.connection.get_state(is_orig) of { KEX_INIT -> kex : SSH2_KEXINIT(length, is_orig); From 868cb5838de5cfbc33aaf9b51892d0d04287238e Mon Sep 17 00:00:00 2001 From: Devin Trejo Date: Fri, 12 Jan 2018 13:53:22 -0500 Subject: [PATCH 02/56] Add smb2_file_sattr --- scripts/policy/protocols/smb/main.bro | 1 + scripts/policy/protocols/smb/smb2-main.bro | 20 +++++++++++++ .../protocol/smb/smb2-com-set-info.pac | 28 +++++++++++++++++++ .../protocol/smb/smb2_com_set_info.bif | 24 ++++++++++++++-- 4 files changed, 71 insertions(+), 2 deletions(-) diff --git a/scripts/policy/protocols/smb/main.bro b/scripts/policy/protocols/smb/main.bro index f94db17f38..51aab775c0 100644 --- a/scripts/policy/protocols/smb/main.bro +++ b/scripts/policy/protocols/smb/main.bro @@ -18,6 +18,7 @@ export { FILE_CLOSE, FILE_DELETE, FILE_RENAME, + FILE_SET_ATTRIBUTE, PIPE_READ, PIPE_WRITE, diff --git a/scripts/policy/protocols/smb/smb2-main.bro b/scripts/policy/protocols/smb/smb2-main.bro index 750a7ff1bc..9faf4b5a8a 100644 --- a/scripts/policy/protocols/smb/smb2-main.bro +++ b/scripts/policy/protocols/smb/smb2-main.bro @@ -231,6 +231,26 @@ event smb2_write_request(c: connection, hdr: SMB2::Header, file_id: SMB2::GUID, SMB::write_file_log(c$smb_state); } +event smb2_file_sattr(c: connection, hdr: SMB2::Header, file_id: SMB2::GUID, times: SMB::MACTimes, attrs: SMB2::FileAttrs) &priority=-5 +{ + SMB::write_file_log(c$smb_state); +} + +event smb2_file_sattr(c: connection, hdr: SMB2::Header, file_id: SMB2::GUID, times: SMB::MACTimes, attrs: SMB2::FileAttrs) &priority=5 +{ + SMB::set_current_file(c$smb_state, file_id$persistent+file_id$volatile); + + switch ( c$smb_state$current_tree$share_type ) + { + case "DISK": + c$smb_state$current_file$action = SMB::FILE_SET_ATTRIBUTE; + break; + default: + c$smb_state$current_file$action = SMB::FILE_SET_ATTRIBUTE; + break; + } +} + event smb2_file_rename(c: connection, hdr: SMB2::Header, file_id: SMB2::GUID, dst_filename: string) &priority=5 { SMB::set_current_file(c$smb_state, file_id$persistent+file_id$volatile); diff --git a/src/analyzer/protocol/smb/smb2-com-set-info.pac b/src/analyzer/protocol/smb/smb2-com-set-info.pac index 379c919d7d..6874808da9 100644 --- a/src/analyzer/protocol/smb/smb2-com-set-info.pac +++ b/src/analyzer/protocol/smb/smb2-com-set-info.pac @@ -6,12 +6,29 @@ enum smb2_set_info_type { }; enum smb_file_info_type { + SMB2_FILE_BASIC_INFO = 0x04, SMB2_FILE_RENAME_INFO = 0x0a, SMB2_FILE_DISPOSITION_INFO = 0x0d, } refine connection SMB_Conn += { + function proc_smb2_set_info_request_file(val: SMB2_file_basic_info): bool + %{ + if ( smb2_file_sattr ) + BifEvent::generate_smb2_file_sattr(bro_analyzer(), + bro_analyzer()->Conn(), + BuildSMB2HeaderVal(${val.sir.header}), + BuildSMB2GUID(${val.sir.file_id}), + SMB_BuildMACTimes(${val.last_write_time}, + ${val.last_access_time}, + ${val.creation_time}, + ${val.change_time}), + smb2_file_attrs_to_bro(${val.file_attrs})); + + return true; + %} + function proc_smb2_set_info_request_file_rename(val: SMB2_file_rename_info): bool %{ if ( smb2_file_rename ) @@ -38,6 +55,16 @@ refine connection SMB_Conn += { }; +type SMB2_file_basic_info(sir: SMB2_set_info_request) = record { + creation_time : SMB_timestamp; + last_access_time : SMB_timestamp; + last_write_time : SMB_timestamp; + change_time : SMB_timestamp; + file_attrs : SMB2_file_attributes; +} &let { + proc: bool = $context.connection.proc_smb2_set_info_request_file(this); +}; + type SMB2_file_rename_info(sir: SMB2_set_info_request) = record { replace_if_exists : uint8; reserved : uint8[7]; @@ -55,6 +82,7 @@ type SMB2_file_disposition_info(sir: SMB2_set_info_request) = record { }; type SMB2_set_info_file_class(sir: SMB2_set_info_request) = case sir.info_level of { + SMB2_FILE_BASIC_INFO -> file_basic : SMB2_file_basic_info(sir); SMB2_FILE_RENAME_INFO -> file_rename : SMB2_file_rename_info(sir); SMB2_FILE_DISPOSITION_INFO -> file_disposition : SMB2_file_disposition_info(sir); default -> info_file_unhandled : empty; diff --git a/src/analyzer/protocol/smb/smb2_com_set_info.bif b/src/analyzer/protocol/smb/smb2_com_set_info.bif index 3aeeb579fe..1f6d9386f8 100644 --- a/src/analyzer/protocol/smb/smb2_com_set_info.bif +++ b/src/analyzer/protocol/smb/smb2_com_set_info.bif @@ -11,7 +11,7 @@ ## ## dst_filename: The filename to rename the file into. ## -## .. bro:see:: smb2_message smb2_file_delete +## .. bro:see:: smb2_message smb2_file_delete smb2_file_sattr event smb2_file_rename%(c: connection, hdr: SMB2::Header, file_id: SMB2::GUID, dst_filename: string%); ## Generated for :abbr:`SMB (Server Message Block)`/:abbr:`CIFS (Common Internet File System)` @@ -23,12 +23,32 @@ event smb2_file_rename%(c: connection, hdr: SMB2::Header, file_id: SMB2::GUID, d ## ## hdr: The parsed header of the :abbr:`SMB (Server Message Block)` version 2 message. ## +## file_id: The SMB2 GUID for the file. +## ## delete_pending: A boolean value to indicate that a file should be deleted ## when it's closed if set to T. ## -## .. bro:see:: smb2_message smb2_file_rename +## .. bro:see:: smb2_message smb2_file_rename smb2_file_sattr event smb2_file_delete%(c: connection, hdr: SMB2::Header, file_id: SMB2::GUID, delete_pending: bool%); +## Generated for :abbr:`SMB (Server Message Block)`/:abbr:`CIFS (Common Internet File System)` +## version 2 requests of type *set_info* of the *file* subtype +## +## For more infomation, see MS-SMB2:2.2.39 +## +## c: The connection. +## +## hdr: The parsed header of the :abbr:`SMB (Server Message Block)` version 2 message. +## +## file_id: The SMB2 GUID for the file. +## +## times: Timestamps associated with the file in question. +## +## attrs: File attributes. +## +## .. bro:see:: smb2_message smb2_file_rename smb2_file_delete +event smb2_file_sattr%(c: connection, hdr: SMB2::Header, file_id: SMB2::GUID, times: SMB::MACTimes, attrs: SMB2::FileAttrs%); + # TODO - Not implemented # Generated for :abbr:`SMB (Server Message Block)`/:abbr:`CIFS (Common Internet File System)` From 17202e9df453a6ad319f0947ef1e29c6927540c1 Mon Sep 17 00:00:00 2001 From: Devin Trejo Date: Wed, 4 Apr 2018 16:55:10 -0400 Subject: [PATCH 03/56] Add unit tests for new SMB2 event -- smb2_file_sattr. --- .../btest/Baseline/scripts.base.protocols.smb.smb2/.stdout | 1 + testing/btest/scripts/base/protocols/smb/smb2.test | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/testing/btest/Baseline/scripts.base.protocols.smb.smb2/.stdout b/testing/btest/Baseline/scripts.base.protocols.smb.smb2/.stdout index bc605dcbb2..299b36fd33 100644 --- a/testing/btest/Baseline/scripts.base.protocols.smb.smb2/.stdout +++ b/testing/btest/Baseline/scripts.base.protocols.smb.smb2/.stdout @@ -5,6 +5,7 @@ smb2_create_response 10.0.0.11 -> 10.0.0.12:445 [file_id=[persistent=73, volatil smb2_create_request 10.0.0.11 -> 10.0.0.12:445 [filename=, disposition=2, create_options=2097185] smb2_create_request 10.0.0.11 -> 10.0.0.12:445 [filename=WP_SMBPlugin.pdf, disposition=2, create_options=68] smb2_create_response 10.0.0.11 -> 10.0.0.12:445 [file_id=[persistent=77, volatile=18446744069414584329], size=0, times=[modified=1323202695.427036, accessed=1323202695.427036, created=1323202695.427036, changed=1323202695.427036], attrs=[read_only=F, hidden=F, system=F, directory=F, archive=T, normal=F, temporary=F, sparse_file=F, reparse_point=F, compressed=F, offline=F, not_content_indexed=F, encrypted=F, integrity_stream=F, no_scrub_data=F], create_action=2] +smb2_file_sattr 10.0.0.11 -> 10.0.0.12:445 [persistent=77, volatile=18446744069414584329] MACTimes:[modified=1319047808.3125, accessed=-1.164447e+10, created=-1.164447e+10, changed=-1.164447e+10] FileAttrs:[read_only=F, hidden=F, system=F, directory=F, archive=F, normal=F, temporary=F, sparse_file=F, reparse_point=F, compressed=F, offline=F, not_content_indexed=F, encrypted=F, integrity_stream=F, no_scrub_data=F] smb2_create_request 10.0.0.11 -> 10.0.0.12:445 [filename=, disposition=1, create_options=32] smb2_create_response 10.0.0.11 -> 10.0.0.12:445 [file_id=[persistent=81, volatile=18446744069414584333], size=8192, times=[modified=1323202695.427036, accessed=1323202695.427036, created=1322343963.945297, changed=1323202695.427036], attrs=[read_only=F, hidden=F, system=F, directory=T, archive=F, normal=F, temporary=F, sparse_file=F, reparse_point=F, compressed=F, offline=F, not_content_indexed=F, encrypted=F, integrity_stream=F, no_scrub_data=F], create_action=1] smb2_create_request 10.0.0.11 -> 10.0.0.12:445 [filename=, disposition=1, create_options=32] diff --git a/testing/btest/scripts/base/protocols/smb/smb2.test b/testing/btest/scripts/base/protocols/smb/smb2.test index 33ce0e29a3..2a05d5edf3 100644 --- a/testing/btest/scripts/base/protocols/smb/smb2.test +++ b/testing/btest/scripts/base/protocols/smb/smb2.test @@ -19,3 +19,9 @@ event smb2_create_response(c: connection, hdr: SMB2::Header, response: SMB2::Cre print fmt("smb2_create_response %s -> %s:%d %s", c$id$orig_h, c$id$resp_h, c$id$resp_p, response); } +event smb2_file_sattr(c: connection, hdr: SMB2::Header, file_id: + SMB2::GUID, times: SMB::MACTimes, attrs: SMB2::FileAttrs) +{ + print fmt("smb2_file_sattr %s -> %s:%d %s MACTimes:%s FileAttrs:%s", c$id$orig_h, c$id$resp_h, c$id$resp_p, file_id, times, attrs); +} + From d9dc8d69d7125474f6eae06718f0b91ddef9e1f2 Mon Sep 17 00:00:00 2001 From: Vern Paxson Date: Thu, 26 Apr 2018 10:51:43 -0700 Subject: [PATCH 04/56] Starting branch for supporting bit operations on count's. This check-in just adds the operators to the lexer and migrates the existing operator names internally. --- src/Expr.cc | 18 ++++++++++-------- src/Expr.h | 3 ++- src/parse.y | 12 ++++++------ src/scan.l | 6 +++--- 4 files changed, 21 insertions(+), 18 deletions(-) diff --git a/src/Expr.cc b/src/Expr.cc index a0456fda31..06fcf448e1 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -24,7 +24,9 @@ const char* expr_name(BroExprTag t) "name", "const", "(*)", "++", "--", "!", "+", "-", - "+", "-", "+=", "-=", "*", "/", "%", "&&", "||", + "+", "-", "+=", "-=", "*", "/", "%", + "&", "|", "^", + "&&", "||", "<", "<=", "==", "!=", ">=", ">", "?:", "ref", "=", "~", "[]", "$", "?$", "[=]", "table()", "set()", "vector()", @@ -777,8 +779,8 @@ Val* BinaryExpr::Fold(Val* v1, Val* v2) const break; - case EXPR_AND: DO_INT_FOLD(&&); break; - case EXPR_OR: DO_INT_FOLD(||); break; + case EXPR_AND_AND: DO_INT_FOLD(&&); break; + case EXPR_OR_OR: DO_INT_FOLD(||); break; case EXPR_LT: DO_INT_VAL_FOLD(<); break; case EXPR_LE: DO_INT_VAL_FOLD(<=); break; @@ -1672,14 +1674,14 @@ Val* BoolExpr::DoSingleEval(Frame* f, Val* v1, Expr* op2) const RE_Matcher* re1 = v1->AsPattern(); RE_Matcher* re2 = v2->AsPattern(); - RE_Matcher* res = tag == EXPR_AND ? + RE_Matcher* res = tag == EXPR_AND_AND ? RE_Matcher_conjunction(re1, re2) : RE_Matcher_disjunction(re1, re2); return new PatternVal(res); } - if ( tag == EXPR_AND ) + if ( tag == EXPR_AND_AND ) { if ( v1->IsZero() ) return v1; @@ -1743,8 +1745,8 @@ Val* BoolExpr::Eval(Frame* f) const VectorVal* result = 0; - // It's either and EXPR_AND or an EXPR_OR. - bool is_and = (tag == EXPR_AND); + // It's either and EXPR_AND_AND or an EXPR_OR_OR. + bool is_and = (tag == EXPR_AND_AND); if ( scalar_v->IsZero() == is_and ) { @@ -1785,7 +1787,7 @@ Val* BoolExpr::Eval(Frame* f) const Val* op2 = vec_v2->Lookup(i); if ( op1 && op2 ) { - bool local_result = (tag == EXPR_AND) ? + bool local_result = (tag == EXPR_AND_AND) ? (! op1->IsZero() && ! op2->IsZero()) : (! op1->IsZero() || ! op2->IsZero()); diff --git a/src/Expr.h b/src/Expr.h index d50506f493..89d44c1cb4 100644 --- a/src/Expr.h +++ b/src/Expr.h @@ -20,7 +20,8 @@ typedef enum { EXPR_INCR, EXPR_DECR, EXPR_NOT, EXPR_POSITIVE, EXPR_NEGATE, EXPR_ADD, EXPR_SUB, EXPR_ADD_TO, EXPR_REMOVE_FROM, EXPR_TIMES, EXPR_DIVIDE, EXPR_MOD, - EXPR_AND, EXPR_OR, + EXPR_AND, EXPR_OR, EXPR_XOR, + EXPR_AND_AND, EXPR_OR_OR, EXPR_LT, EXPR_LE, EXPR_EQ, EXPR_NE, EXPR_GE, EXPR_GT, EXPR_COND, EXPR_REF, diff --git a/src/parse.y b/src/parse.y index 89bb5adbee..90d4c71309 100644 --- a/src/parse.y +++ b/src/parse.y @@ -34,8 +34,8 @@ %left ',' '|' %right '=' TOK_ADD_TO TOK_REMOVE_FROM %right '?' ':' -%left TOK_OR -%left TOK_AND +%left TOK_OR_OR +%left TOK_AND_AND %nonassoc TOK_HOOK %nonassoc '<' '>' TOK_LE TOK_GE TOK_EQ TOK_NE %left TOK_IN TOK_NOT_IN @@ -388,16 +388,16 @@ expr: $$ = new ModExpr($1, $3); } - | expr TOK_AND expr + | expr TOK_AND_AND expr { set_location(@1, @3); - $$ = new BoolExpr(EXPR_AND, $1, $3); + $$ = new BoolExpr(EXPR_AND_AND, $1, $3); } - | expr TOK_OR expr + | expr TOK_OR_OR expr { set_location(@1, @3); - $$ = new BoolExpr(EXPR_OR, $1, $3); + $$ = new BoolExpr(EXPR_OR_OR, $1, $3); } | expr TOK_EQ expr diff --git a/src/scan.l b/src/scan.l index 05baea5735..e848b78a43 100644 --- a/src/scan.l +++ b/src/scan.l @@ -193,7 +193,7 @@ ESCSEQ (\\([^\n]|[0-7]+|x[[:xdigit:]]+)) RET_CONST(new SubNetVal(IPPrefix(IPAddr(ip), len))) } -[!%*/+\-,:;<=>?()\[\]{}~$|] return yytext[0]; +[!%*/+\-,:;<=>?()\[\]{}~$|&^] return yytext[0]; "--" return TOK_DECR; "++" return TOK_INCR; @@ -206,8 +206,8 @@ ESCSEQ (\\([^\n]|[0-7]+|x[[:xdigit:]]+)) ">=" return TOK_GE; "<=" return TOK_LE; -"&&" return TOK_AND; -"||" return TOK_OR; +"&&" return TOK_AND_AND; +"||" return TOK_OR_OR; add return TOK_ADD; addr return TOK_ADDR; From 1658931af1f5c35533457d15735c65462d1e8af1 Mon Sep 17 00:00:00 2001 From: Vern Paxson Date: Thu, 26 Apr 2018 12:38:24 -0700 Subject: [PATCH 05/56] bitwise operations for "count" types implemented --- src/Expr.cc | 85 ++++++++++++++++++++++++++++++++++++++++++++++- src/Expr.h | 28 +++++++++++++++- src/SerialObj.h | 2 +- src/SerialTypes.h | 2 ++ src/parse.y | 33 ++++++++++++++++-- 5 files changed, 144 insertions(+), 6 deletions(-) diff --git a/src/Expr.cc b/src/Expr.cc index 06fcf448e1..5f79d08dbe 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -23,7 +23,7 @@ const char* expr_name(BroExprTag t) static const char* expr_names[int(NUM_EXPRS)] = { "name", "const", "(*)", - "++", "--", "!", "+", "-", + "++", "--", "!", "~", "+", "-", "+", "-", "+=", "-=", "*", "/", "%", "&", "|", "^", "&&", "||", @@ -706,6 +706,12 @@ Val* BinaryExpr::Fold(Val* v1, Val* v2) const else \ Internal("bad type in BinaryExpr::Fold"); +#define DO_UINT_FOLD(op) \ + if ( is_unsigned ) \ + u3 = u1 op u2; \ + else \ + Internal("bad type in BinaryExpr::Fold"); + #define DO_FOLD(op) \ if ( is_integral ) \ i3 = i1 op i2; \ @@ -779,6 +785,10 @@ Val* BinaryExpr::Fold(Val* v1, Val* v2) const break; + case EXPR_AND: DO_UINT_FOLD(&); break; + case EXPR_OR: DO_UINT_FOLD(|); break; + case EXPR_XOR: DO_UINT_FOLD(^); break; + case EXPR_AND_AND: DO_INT_FOLD(&&); break; case EXPR_OR_OR: DO_INT_FOLD(||); break; @@ -1081,6 +1091,39 @@ bool IncrExpr::DoUnserialize(UnserialInfo* info) return true; } +ComplementExpr::ComplementExpr(Expr* arg_op) : UnaryExpr(EXPR_COMPLEMENT, arg_op) + { + if ( IsError() ) + return; + + BroType* t = op->Type(); + TypeTag bt = t->Tag(); + + if ( bt != TYPE_COUNT ) + ExprError("requires \"count\" operand"); + else + SetType(base_type(TYPE_COUNT)); + } + +Val* ComplementExpr::Fold(Val* v) const + { + return new Val(~ v->InternalUnsigned(), type->Tag()); + } + +IMPLEMENT_SERIAL(ComplementExpr, SER_COMPLEMENT_EXPR); + +bool ComplementExpr::DoSerialize(SerialInfo* info) const + { + DO_SERIALIZE(SER_COMPLEMENT_EXPR, UnaryExpr); + return true; + } + +bool ComplementExpr::DoUnserialize(UnserialInfo* info) + { + DO_UNSERIALIZE(UnaryExpr); + return true; + } + NotExpr::NotExpr(Expr* arg_op) : UnaryExpr(EXPR_NOT, arg_op) { if ( IsError() ) @@ -1817,6 +1860,46 @@ bool BoolExpr::DoUnserialize(UnserialInfo* info) return true; } +BitExpr::BitExpr(BroExprTag arg_tag, Expr* arg_op1, Expr* arg_op2) +: BinaryExpr(arg_tag, arg_op1, arg_op2) + { + if ( IsError() ) + return; + + TypeTag bt1 = op1->Type()->Tag(); + if ( IsVector(bt1) ) + bt1 = op1->Type()->AsVectorType()->YieldType()->Tag(); + + TypeTag bt2 = op2->Type()->Tag(); + if ( IsVector(bt2) ) + bt2 = op2->Type()->AsVectorType()->YieldType()->Tag(); + + if ( bt1 == TYPE_COUNT && bt2 == bt1 ) + { + if ( is_vector(op1) || is_vector(op2) ) + SetType(new VectorType(base_type(bt1))); + else + SetType(base_type(bt1)); + } + + else + ExprError("requires \"count\" operands"); + } + +IMPLEMENT_SERIAL(BitExpr, SER_BIT_EXPR); + +bool BitExpr::DoSerialize(SerialInfo* info) const + { + DO_SERIALIZE(SER_BIT_EXPR, BinaryExpr); + return true; + } + +bool BitExpr::DoUnserialize(UnserialInfo* info) + { + DO_UNSERIALIZE(BinaryExpr); + return true; + } + EqExpr::EqExpr(BroExprTag arg_tag, Expr* arg_op1, Expr* arg_op2) : BinaryExpr(arg_tag, arg_op1, arg_op2) { diff --git a/src/Expr.h b/src/Expr.h index 89d44c1cb4..8e8b6cc96b 100644 --- a/src/Expr.h +++ b/src/Expr.h @@ -17,7 +17,9 @@ typedef enum { EXPR_ANY = -1, EXPR_NAME, EXPR_CONST, EXPR_CLONE, - EXPR_INCR, EXPR_DECR, EXPR_NOT, EXPR_POSITIVE, EXPR_NEGATE, + EXPR_INCR, EXPR_DECR, + EXPR_NOT, EXPR_COMPLEMENT, + EXPR_POSITIVE, EXPR_NEGATE, EXPR_ADD, EXPR_SUB, EXPR_ADD_TO, EXPR_REMOVE_FROM, EXPR_TIMES, EXPR_DIVIDE, EXPR_MOD, EXPR_AND, EXPR_OR, EXPR_XOR, @@ -378,6 +380,19 @@ protected: DECLARE_SERIAL(IncrExpr); }; +class ComplementExpr : public UnaryExpr { +public: + explicit ComplementExpr(Expr* op); + +protected: + friend class Expr; + ComplementExpr() { } + + Val* Fold(Val* v) const override; + + DECLARE_SERIAL(ComplementExpr); +}; + class NotExpr : public UnaryExpr { public: explicit NotExpr(Expr* op); @@ -531,6 +546,17 @@ protected: DECLARE_SERIAL(BoolExpr); }; +class BitExpr : public BinaryExpr { +public: + BitExpr(BroExprTag tag, Expr* op1, Expr* op2); + +protected: + friend class Expr; + BitExpr() { } + + DECLARE_SERIAL(BitExpr); +}; + class EqExpr : public BinaryExpr { public: EqExpr(BroExprTag tag, Expr* op1, Expr* op2); diff --git a/src/SerialObj.h b/src/SerialObj.h index 77dc28aefd..b502414f71 100644 --- a/src/SerialObj.h +++ b/src/SerialObj.h @@ -3,7 +3,7 @@ // How to make objects of class Foo serializable: // // 1. Derive Foo (directly or indirectly) from SerialObj. -// 2. Add a SER_FOO constant to SerialTypes below. +// 2. Add a SER_FOO constant to SerialTypes in SerialTypes.h. // 3. Add DECLARE_SERIAL(Foo) into class definition. // 4. Add a (preferably protected) default ctor if it doesn't already exist. // 5. For non-abstract classes, add IMPLEMENT_SERIAL(Foo, SER_FOO) to *.cc diff --git a/src/SerialTypes.h b/src/SerialTypes.h index 8a1a2abf51..c5b25d32fb 100644 --- a/src/SerialTypes.h +++ b/src/SerialTypes.h @@ -162,6 +162,8 @@ SERIAL_EXPR(SET_CONSTRUCTOR_EXPR, 41) SERIAL_EXPR(VECTOR_CONSTRUCTOR_EXPR, 42) SERIAL_EXPR(TABLE_COERCE_EXPR, 43) SERIAL_EXPR(VECTOR_COERCE_EXPR, 44) +SERIAL_EXPR(BIT_EXPR, 45) +SERIAL_EXPR(COMPLEMENT_EXPR, 46) #define SERIAL_STMT(name, val) SERIAL_CONST(name, val, STMT) SERIAL_STMT(STMT, 1) diff --git a/src/parse.y b/src/parse.y index 90d4c71309..6f7a43ae7f 100644 --- a/src/parse.y +++ b/src/parse.y @@ -31,7 +31,7 @@ %token TOK_NO_TEST -%left ',' '|' +%left ',' %right '=' TOK_ADD_TO TOK_REMOVE_FROM %right '?' ':' %left TOK_OR_OR @@ -39,10 +39,13 @@ %nonassoc TOK_HOOK %nonassoc '<' '>' TOK_LE TOK_GE TOK_EQ TOK_NE %left TOK_IN TOK_NOT_IN +%left '|' +%left '^' +%left '&' %left '+' '-' %left '*' '/' '%' %left TOK_INCR TOK_DECR -%right '!' +%right '!' '~' %left '$' '[' ']' '(' ')' TOK_HAS_FIELD TOK_HAS_ATTR %type opt_no_test opt_no_test_block opt_deprecated @@ -334,6 +337,12 @@ expr: $$ = new NotExpr($2); } + | '~' expr + { + set_location(@1, @2); + $$ = new ComplementExpr($2); + } + | '-' expr %prec '!' { set_location(@1, @2); @@ -388,6 +397,24 @@ expr: $$ = new ModExpr($1, $3); } + | expr '&' expr + { + set_location(@1, @3); + $$ = new BitExpr(EXPR_AND, $1, $3); + } + + | expr '|' expr + { + set_location(@1, @3); + $$ = new BitExpr(EXPR_OR, $1, $3); + } + + | expr '^' expr + { + set_location(@1, @3); + $$ = new BitExpr(EXPR_XOR, $1, $3); + } + | expr TOK_AND_AND expr { set_location(@1, @3); @@ -700,7 +727,7 @@ expr: $$ = new ConstExpr(new PatternVal($1)); } - | '|' expr '|' + | '|' expr '|' %prec '(' { set_location(@1, @3); $$ = new SizeExpr($2); From 58ffd61dcc2874d07089b325850c670d6f622e14 Mon Sep 17 00:00:00 2001 From: Vern Paxson Date: Thu, 26 Apr 2018 13:25:04 -0700 Subject: [PATCH 06/56] test suite for bitwise operators brief NEWS blurb allow for "counter" operands (does anyone still use these?) for one (but not both) of the bitwise operands --- NEWS | 4 ++++ src/Expr.cc | 11 +++++++---- testing/btest/Baseline/language.count/out | 11 +++++++++++ testing/btest/language/count.bro | 11 +++++++++++ 4 files changed, 33 insertions(+), 4 deletions(-) diff --git a/NEWS b/NEWS index aa43682047..eeeda2a42c 100644 --- a/NEWS +++ b/NEWS @@ -146,6 +146,10 @@ New Functionality - Added new SMB events: smb1_transaction_secondary_request, smb1_transaction2_secondary_request, smb1_transaction_response +- Added support for bitwise operations on "count" values. '&', '|' and + '^' are binary "and", "or" and "xor" operators, and '~' is a unary + ones-complement operator. + Changed Functionality --------------------- diff --git a/src/Expr.cc b/src/Expr.cc index 5f79d08dbe..f5c8e66b50 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -1874,12 +1874,15 @@ BitExpr::BitExpr(BroExprTag arg_tag, Expr* arg_op1, Expr* arg_op2) if ( IsVector(bt2) ) bt2 = op2->Type()->AsVectorType()->YieldType()->Tag(); - if ( bt1 == TYPE_COUNT && bt2 == bt1 ) + if ( (bt1 == TYPE_COUNT || bt1 == TYPE_COUNTER) && + (bt2 == TYPE_COUNT || bt2 == TYPE_COUNTER) ) { - if ( is_vector(op1) || is_vector(op2) ) - SetType(new VectorType(base_type(bt1))); + if ( bt1 == TYPE_COUNTER && bt2 == TYPE_COUNTER ) + ExprError("cannot apply a bitwise operator to two \"counter\" operands"); + else if ( is_vector(op1) || is_vector(op2) ) + SetType(new VectorType(base_type(TYPE_COUNT))); else - SetType(base_type(bt1)); + SetType(base_type(TYPE_COUNT)); } else diff --git a/testing/btest/Baseline/language.count/out b/testing/btest/Baseline/language.count/out index 4ef65b6098..f1e1eef587 100644 --- a/testing/btest/Baseline/language.count/out +++ b/testing/btest/Baseline/language.count/out @@ -14,5 +14,16 @@ modulus operator (PASS) division operator (PASS) assignment operator (PASS) assignment operator (PASS) +bitwise and (PASS) +bitwise and (PASS) +bitwise and (PASS) +bitwise or (PASS) +bitwise or (PASS) +bitwise or (PASS) +bitwise xor (PASS) +bitwise xor (PASS) +bitwise xor (PASS) +bitwise complement (PASS) +bitwise complement (PASS) max count value = 18446744073709551615 (PASS) max count value = 18446744073709551615 (PASS) diff --git a/testing/btest/language/count.bro b/testing/btest/language/count.bro index b0972e29fa..39a3786dfb 100644 --- a/testing/btest/language/count.bro +++ b/testing/btest/language/count.bro @@ -47,6 +47,17 @@ event bro_init() test_case( "assignment operator", c2 == 8 ); c2 -= 2; test_case( "assignment operator", c2 == 6 ); + test_case( "bitwise and", c2 & 0x4 == 0x4 ); + test_case( "bitwise and", c4 & 0x4 == 0x4 ); + test_case( "bitwise and", c8 & 0x4 == 0x0 ); + test_case( "bitwise or", c2 | 0x4 == c2 ); + test_case( "bitwise or", c4 | 0x4 == c4 ); + test_case( "bitwise or", c8 | 0x4 == c7 ); + test_case( "bitwise xor", c7 ^ 0x4 == c8 ); + test_case( "bitwise xor", c4 ^ 0x4 == 251 ); + test_case( "bitwise xor", c8 ^ 0x4 == c7 ); + test_case( "bitwise complement", ~c6 == 0 ); + test_case( "bitwise complement", ~~c4 == c4 ); # Max. value tests From 1ca5a5a16dd6e774b1173006c26c8e7b68c6cfb4 Mon Sep 17 00:00:00 2001 From: Vern Paxson Date: Thu, 26 Apr 2018 14:57:55 -0700 Subject: [PATCH 07/56] document the '|' operator for patterns --- doc/script-reference/types.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/script-reference/types.rst b/doc/script-reference/types.rst index 651ebfb411..3d6f74aef0 100644 --- a/doc/script-reference/types.rst +++ b/doc/script-reference/types.rst @@ -233,6 +233,13 @@ Here is a more detailed description of each type: is false since "oob" does not appear at the start of "foobar". The ``!in`` operator would yield the negation of ``in``. + Finally, you can create a disjunction (either-or) of two patterns + using the ``|`` operator. For example:: + + /foo/ | /bar/ in "foobar" + + yields true, like in the similar example above. + .. bro:type:: port A type representing transport-level port numbers (besides TCP and From 2bd3a7664df2c5d3200d782ae4ec2dfccd71bc89 Mon Sep 17 00:00:00 2001 From: Vern Paxson Date: Thu, 26 Apr 2018 16:22:17 -0700 Subject: [PATCH 08/56] documentation for bitwise operators --- doc/script-reference/types.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/script-reference/types.rst b/doc/script-reference/types.rst index 3d6f74aef0..fe77ece90c 100644 --- a/doc/script-reference/types.rst +++ b/doc/script-reference/types.rst @@ -91,6 +91,10 @@ Here is a more detailed description of each type: type, but a unary plus or minus applied to a "count" results in an "int". + In addition, "count" types support bitwise operations. You can use + ``&``, ``|``, and ``^`` for bitwise ``and'', ``or'', and ``xor''. You + can also use ``~`` for bitwise (one's) complement. + .. bro:type:: double A numeric type representing a double-precision floating-point @@ -592,6 +596,9 @@ Here is a more detailed description of each type: The resulting vector of bool is the logical "and" (or logical "or") of each element of the operand vectors. + Vectors of type ``count`` can also be operands for the bitwise and/or/xor + operators, ``&``, ``|`` and ``^``. + See the :bro:keyword:`for` statement for info on how to iterate over the elements in a vector. From 98a430c1eb6ff7af91725076a90929bd935e65ac Mon Sep 17 00:00:00 2001 From: Julien Wallior Date: Wed, 15 Mar 2017 12:11:10 -0400 Subject: [PATCH 09/56] Decrypt the krb ticket and send authentication data out. --- scripts/base/init-bare.bro | 3 + src/analyzer/protocol/krb/CMakeLists.txt | 1 + src/analyzer/protocol/krb/KRB.cc | 100 ++++++++++++++++++++- src/analyzer/protocol/krb/KRB.h | 8 ++ src/analyzer/protocol/krb/KRB_TCP.h | 2 + src/analyzer/protocol/krb/krb-analyzer.pac | 5 +- src/analyzer/protocol/krb/krb.pac | 10 ++- src/analyzer/protocol/krb/krb_TCP.pac | 10 ++- src/analyzer/protocol/krb/types.bif | 2 + 9 files changed, 137 insertions(+), 4 deletions(-) diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index bd2206ceab..9b2f58ec46 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -4222,6 +4222,7 @@ export { module KRB; export { + const keytab = "/etc/krb5.keytab" &redef; ## KDC Options. See :rfc:`4120` type KRB::KDC_Options: record { ## The ticket to be issued should have its forwardable flag set. @@ -4344,6 +4345,8 @@ export { cipher : count; ## Cipher text of the ticket ciphertext : string &optional; + ## Authentication info + authenticationinfo: string &optional; }; type KRB::Ticket_Vector: vector of KRB::Ticket; diff --git a/src/analyzer/protocol/krb/CMakeLists.txt b/src/analyzer/protocol/krb/CMakeLists.txt index 1cac35d626..95f6d672a1 100644 --- a/src/analyzer/protocol/krb/CMakeLists.txt +++ b/src/analyzer/protocol/krb/CMakeLists.txt @@ -9,6 +9,7 @@ bro_plugin_cc(KRB.cc) bro_plugin_cc(KRB_TCP.cc) bro_plugin_bif(types.bif) bro_plugin_bif(events.bif) +bro_plugin_link_library(-lkrb5) bro_plugin_pac(krb.pac krb-protocol.pac krb-analyzer.pac krb-asn1.pac krb-defs.pac diff --git a/src/analyzer/protocol/krb/KRB.cc b/src/analyzer/protocol/krb/KRB.cc index ba7dbd9ccd..f080efa7f0 100644 --- a/src/analyzer/protocol/krb/KRB.cc +++ b/src/analyzer/protocol/krb/KRB.cc @@ -7,13 +7,47 @@ using namespace analyzer::krb; KRB_Analyzer::KRB_Analyzer(Connection* conn) - : Analyzer("KRB", conn) + : Analyzer("KRB", conn), + krb_available(false) { interp = new binpac::KRB::KRB_Conn(this); + + const char* keytab_filename = BifConst::KRB::keytab->CheckString(); + if (access(keytab_filename, R_OK) != 0) + { + reporter->Warning("KRB: Can't access keytab (%s)", keytab_filename); + return; + } + + krb5_error_code retval = krb5_init_context(&krb_context); + if (retval) + { + reporter->Warning("KRB: Couldn't initialize the context (%s)", krb5_get_error_message(krb_context, retval)); + } + else + { + krb_available = true; + + retval = krb5_kt_resolve(krb_context, keytab_filename, &krb_keytab); + if (retval) + { + reporter->Warning("KRB: Couldn't resolve keytab (%s)", krb5_get_error_message(krb_context, retval)); + } + } + } KRB_Analyzer::~KRB_Analyzer() { + if (krb_available) + { + krb5_error_code retval = krb5_kt_close(krb_context, krb_keytab); + if (retval) + { + reporter->Warning("KRB: Couldn't close keytab (%s)", krb5_get_error_message(krb_context, retval)); + } + krb5_free_context(krb_context); + } delete interp; } @@ -37,3 +71,67 @@ void KRB_Analyzer::DeliverPacket(int len, const u_char* data, bool orig, } } +StringVal* KRB_Analyzer::GetAuthenticationInfo(const BroString* principal, const BroString* ciphertext, const bro_uint_t enctype) + { + StringVal* ret = new StringVal("nouser"); + if (!krb_available) + { + return ret; + } + + BroString delim("/"); + int pos = principal->FindSubstring(&delim); + if (pos == -1) + { + reporter->Warning("KRB: Couldn't parse principal (%s)", principal->CheckString()); + return ret; + } + BroString* service(principal->GetSubstring(0, pos)); + BroString* hostname(principal->GetSubstring(pos + 1, -1)); + if (!service || !hostname) + { + reporter->Warning("KRB: Couldn't parse principal (%s)", principal->CheckString()); + if (!service) + free(service); + if (!hostname) + free(hostname); + return ret; + } + krb5_principal sprinc; + krb5_error_code retval = krb5_sname_to_principal(krb_context, reinterpret_cast(hostname->Bytes()), reinterpret_cast(service->Bytes()), KRB5_NT_SRV_HST, &sprinc); + if (retval) + { + reporter->Warning("KRB: Couldn't generate principal name (%s)", krb5_get_error_message(krb_context, retval)); + return ret; + } + free(service); + free(hostname); + + krb5_ticket tkt; + tkt.server = sprinc; + tkt.enc_part.enctype = enctype; + tkt.enc_part.ciphertext.data = reinterpret_cast(ciphertext->Bytes()); + tkt.enc_part.ciphertext.length = ciphertext->Len(); + + retval = krb5_server_decrypt_ticket_keytab(krb_context, krb_keytab, &tkt); + if (retval) + { + reporter->Warning("KRB: Couldn't decrypt ticket (%s)", krb5_get_error_message(krb_context, retval)); + return ret; + } + + char* cp; + retval = krb5_unparse_name(krb_context, tkt.enc_part2->client, &cp); + if (retval) + { + reporter->Warning("KRB: Couldn't unparse name (%s)", krb5_get_error_message(krb_context, retval)); + return ret; + } + free(ret); + ret = new StringVal(cp); + + krb5_free_unparsed_name(krb_context, cp); + + return ret; + + } diff --git a/src/analyzer/protocol/krb/KRB.h b/src/analyzer/protocol/krb/KRB.h index 99e0529ff1..52cf9fa2bb 100644 --- a/src/analyzer/protocol/krb/KRB.h +++ b/src/analyzer/protocol/krb/KRB.h @@ -5,6 +5,8 @@ #include "krb_pac.h" +#include + namespace analyzer { namespace krb { class KRB_Analyzer : public analyzer::Analyzer { @@ -20,9 +22,15 @@ public: static analyzer::Analyzer* Instantiate(Connection* conn) { return new KRB_Analyzer(conn); } + StringVal* GetAuthenticationInfo(const BroString* principal, const BroString* ciphertext, const bro_uint_t enctype); + protected: binpac::KRB::KRB_Conn* interp; + + bool krb_available; + krb5_context krb_context; + krb5_keytab krb_keytab; }; } } // namespace analyzer::* diff --git a/src/analyzer/protocol/krb/KRB_TCP.h b/src/analyzer/protocol/krb/KRB_TCP.h index 0ce4d5f65d..f6c679be63 100644 --- a/src/analyzer/protocol/krb/KRB_TCP.h +++ b/src/analyzer/protocol/krb/KRB_TCP.h @@ -22,6 +22,8 @@ public: // Overriden from tcp::TCP_ApplicationAnalyzer. void EndpointEOF(bool is_orig) override; + StringVal* GetAuthenticationInfo(const BroString* principal, const BroString* ciphertext, const bro_uint_t enctype) { return new StringVal(""); } + static analyzer::Analyzer* Instantiate(Connection* conn) { return new KRB_Analyzer(conn); } diff --git a/src/analyzer/protocol/krb/krb-analyzer.pac b/src/analyzer/protocol/krb/krb-analyzer.pac index 6390fb8fd0..2af70fff98 100644 --- a/src/analyzer/protocol/krb/krb-analyzer.pac +++ b/src/analyzer/protocol/krb/krb-analyzer.pac @@ -245,8 +245,11 @@ refine connection KRB_Conn += { rv->Assign(0, new Val(${msg.ap_options.use_session_key}, TYPE_BOOL)); rv->Assign(1, new Val(${msg.ap_options.mutual_required}, TYPE_BOOL)); + RecordVal* rvticket = proc_ticket(${msg.ticket}); + StringVal* authenticationinfo = bro_analyzer()->GetAuthenticationInfo(rvticket->Lookup(2)->AsString(), rvticket->Lookup(4)->AsString(), rvticket->Lookup(3)->AsCount()); + rvticket->Assign(5, authenticationinfo); BifEvent::generate_krb_ap_request(bro_analyzer(), bro_analyzer()->Conn(), - proc_ticket(${msg.ticket}), rv); + rvticket, rv); } return true; %} diff --git a/src/analyzer/protocol/krb/krb.pac b/src/analyzer/protocol/krb/krb.pac index 508fb78a7a..9a3b290ad1 100644 --- a/src/analyzer/protocol/krb/krb.pac +++ b/src/analyzer/protocol/krb/krb.pac @@ -4,14 +4,22 @@ %extern{ #include "types.bif.h" #include "events.bif.h" + +namespace analyzer { namespace krb { class KRB_Analyzer; } } +namespace binpac { namespace KRB { class KRB_Conn; } } +typedef analyzer::krb::KRB_Analyzer* KRBAnalyzer; + +#include "KRB.h" %} +extern type KRBAnalyzer; + analyzer KRB withcontext { connection: KRB_Conn; flow: KRB_Flow; }; -connection KRB_Conn(bro_analyzer: BroAnalyzer) { +connection KRB_Conn(bro_analyzer: KRBAnalyzer) { upflow = KRB_Flow(true); downflow = KRB_Flow(false); }; diff --git a/src/analyzer/protocol/krb/krb_TCP.pac b/src/analyzer/protocol/krb/krb_TCP.pac index 6748c5fcbb..6611a549e4 100644 --- a/src/analyzer/protocol/krb/krb_TCP.pac +++ b/src/analyzer/protocol/krb/krb_TCP.pac @@ -4,14 +4,22 @@ %extern{ #include "types.bif.h" #include "events.bif.h" + +namespace analyzer { namespace krb_tcp { class KRB_Analyzer; } } +namespace binpac { namespace KRB_TCP { class KRB_Conn; } } +typedef analyzer::krb_tcp::KRB_Analyzer* KRBTCPAnalyzer; + +#include "KRB_TCP.h" %} +extern type KRBTCPAnalyzer; + analyzer KRB_TCP withcontext { connection: KRB_Conn; flow: KRB_Flow; }; -connection KRB_Conn(bro_analyzer: BroAnalyzer) { +connection KRB_Conn(bro_analyzer: KRBTCPAnalyzer) { upflow = KRB_Flow(true); downflow = KRB_Flow(false); }; diff --git a/src/analyzer/protocol/krb/types.bif b/src/analyzer/protocol/krb/types.bif index 8393adbf3c..11a80781f8 100644 --- a/src/analyzer/protocol/krb/types.bif +++ b/src/analyzer/protocol/krb/types.bif @@ -1,5 +1,7 @@ module KRB; +const keytab: string; + type Error_Msg: record; type SAFE_Msg: record; From ae7625bb2bc67bea5881a912465667ce8732b4d1 Mon Sep 17 00:00:00 2001 From: Julien Wallior Date: Wed, 4 Apr 2018 15:57:15 -0400 Subject: [PATCH 10/56] Fix base.protocols.krb.kinit unit test. --- testing/btest/Baseline/scripts.base.protocols.krb.kinit/output | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/btest/Baseline/scripts.base.protocols.krb.kinit/output b/testing/btest/Baseline/scripts.base.protocols.krb.kinit/output index c42038b5d0..50876c8d47 100644 --- a/testing/btest/Baseline/scripts.base.protocols.krb.kinit/output +++ b/testing/btest/Baseline/scripts.base.protocols.krb.kinit/output @@ -1,3 +1,3 @@ KRB_AP_REQUEST -[pvno=5, realm=VLADG.NET, service_name=krbtgt/VLADG.NET, cipher=18, ciphertext={\x9fY\xd0f\x8dS\xf4I\x88\x04\xfa\xc1\xd8m\xa2\xb7+\xbb\x19\xcag\x0c\x13\xd1g*\xfc\x18\xd1\xb1\x80!\xbd\x85\xec\xf9\x9b\xfa-\x18\xb6\xf5h\x91\xe7\x99\xf4\xdb\x93\xa0\xc7\x90\x1e\xa9\x95v\xd3\x12\xfa,9\x1d\x0b\xd0\xa1\xd25\x0f\x1f[G\xdf\xd0\xbbd\x06$2\xd1\xae\x130qZiY\x07@\xe9\xf9\xff\xa4\x9a\xd4\x09\xf0\x0d\xc1R\x10M\xbdKOV\xfd\xf6\x13\xf6\x9a\x95N\xdf!\xf6x\x94\xd8j\xa5\xdcp\xa8\x04\x99\x02x\xdb$\xd8\xfa_o\x8dV\xc8\x0a\xfe\x00\xf3&c\x0c8\xd1\xd0\xe9\x8e\xab\xfe&\xfe\x00\x8d$\x98I\xe5\x8d\x94rM4%\xd8\xfe\xa9\x08\x06\xc6\x95H7\xf7HCq\xb9\x0d$\x95?\x83B\x82\xdd\xea\xc3f3\xcc\xbb\x09\x0d-\x09;\xa6i%\xcd\xba\x11\xd4\xe0\x12w\xd0G&\xdaj\x82\x7f;\xf3\x1d\x10\xa4l\x06\x16l\x1bc\xa1\xd1\x15!\x00\x8a\xff\x8a\x06\xe7U^: Date: Tue, 8 May 2018 14:46:35 -0400 Subject: [PATCH 11/56] Add krb unit test --- .../.stdout | 1 + testing/btest/Traces/krb/smb2_krb.keytab | Bin 0 -> 102 bytes testing/btest/Traces/krb/smb2_krb.pcap | Bin 0 -> 44485 bytes .../scripts/base/protocols/krb/smb2_krb.test | 19 ++++++++++++++++++ 4 files changed, 20 insertions(+) create mode 100644 testing/btest/Baseline/scripts.base.protocols.krb.smb2_krb/.stdout create mode 100644 testing/btest/Traces/krb/smb2_krb.keytab create mode 100755 testing/btest/Traces/krb/smb2_krb.pcap create mode 100644 testing/btest/scripts/base/protocols/krb/smb2_krb.test diff --git a/testing/btest/Baseline/scripts.base.protocols.krb.smb2_krb/.stdout b/testing/btest/Baseline/scripts.base.protocols.krb.smb2_krb/.stdout new file mode 100644 index 0000000000..cd2430defe --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.krb.smb2_krb/.stdout @@ -0,0 +1 @@ +wallior@DS.SUSQ.COM diff --git a/testing/btest/Traces/krb/smb2_krb.keytab b/testing/btest/Traces/krb/smb2_krb.keytab new file mode 100644 index 0000000000000000000000000000000000000000..0f637c1ef07b1df7d1cd11de45d1054333a606e9 GIT binary patch literal 102 zcmZQ&VqjoMU|?e4b_v!C4h;^}bN2UTV9CfYE@6-;$f(FkOiD`3u`n{!ODWbXE-fz9 zOU};)YGI7nZ!5;cAjF{Xq9ofgF)lmD+Ua=L(bl^++r8R3`l6o;C;Kiv5g>9|9;lcJ E0A=tV-2eap literal 0 HcmV?d00001 diff --git a/testing/btest/Traces/krb/smb2_krb.pcap b/testing/btest/Traces/krb/smb2_krb.pcap new file mode 100755 index 0000000000000000000000000000000000000000..0f726f45d7b9d2e0c5d69a54c67ddcb1d57019f9 GIT binary patch literal 44485 zcmd6Q2V51$^Y|V-P{D#7D^H_XLBNL31}gS~6)XWk5wQRwDq?$L*JvzJQDej!do*@q z&{(3V6g#$95)Jl-B`Wxv*}Hk=c;$@W|M&Y}K6`U}x3@DpJ3BMm9v6@OyxNAT7=9pihKsgxm&{sYDET?akjK0!PVywTD zz~=sqG+@B8>?b;~OK^`dPgy6!Q?aadpG=nJEU;fwupGAHQ}<3SV(`wn`zZtzK7IO> z%kSfujekpZXTKf+UAlEp`*-WorkigkwWqrWv+3yDxsz{aej5-GIWTI>NcEto7vuq?$}A~;qD2Kfe5viJI^8BHgDdX zSJDA0*#PLkn1}tfzBXsftHs`vNtmw@r9kOt7)xx6{R22gs!o z5*pgrIIOMdQ9a4ET$RYE*x-R-gCZis1_nn&26t=SvYux{Z=dL4VId>Kf+NBsqhi8P zsD2+?h^+x!81o3#cj0MVKs{wbwKqu9G|=CfeM;Gbbh5$5oZ`>;g{mZpmi~q8t_ri! zad%{q&gGvM{k!Pw>T$KsY+IZ~S#4~bcn3`diY-XUYAzuMjoY?qU}nJW$|NB(q29Zz z3zj5g4VREhCQDHuAxi6w65^ylLf)@R`JYG#a_Sp*fjzWiJjx}+?nZbzO311#V+pxk zIY`s2iN6!e9k2!4&taP=AtGM{=0c4M?Jfpu5>f_qK5;dW^>m;D`X@Ma)%B7D2S1Ml z2j3(Q4|l6QrJ}t-uUV7e;HXM)$Oom_8gwjyMJ5z@>5)+2X_BptV7D4Bh1cY>wPR{P zzcwNN1RD(Bd$yLIc;QWbWF1+Z`A*KG=qFH_} zZ22_DfFkSH9c`eW0Q+2L#W?K$4A z@K;$~hTZx)MKdDAeLz6D-tIB?Tyxp>XRfgR=Vb|@e>wXKeUe zf3i0T(>b6={YTBqe0m8sf=MP5O#5BH{ zcy-D7f=5Q(EIXoCLwDQl>vlTajlFujeZ2-<3ZJ>~Y544G^R`q_&nxYrZuInMpM*tA zO7;46N!R*W*{_CH=xtLzH8ZHAz3rxq-!4vUd3#Mrm;H{Po~)Y}vj6*WZCiFN{`1tF zC-?0ho|$!W^0DCl_dH`>H#py|&&l!uuie_%?kv&j=Ap@df4iyuPkwhERhT%t&-fb2 z`!lEYEMBkdxbYR2bRS=S+{1Mlo6nbg{+sW(b`gIRz8+Spz_@Ewub#Q@k>syVt}y$L ze&s6t?)O`(D}`7SkK?x!(^b`D-joWf`*vr^JDJZ0W!Ju0Zei?>9tFcjCrtk9_QyKe zhw8k4SM}7%(48;7`lrkLi5>R0+?yQz{`*%Yqdc0Pon7icd(`|3LGxR%v}?Jg<&HIK zf6*0g)+96F-erF$pNr^=VJXX?l)7L)Fy^rzRl_sTekA>1Y(MT@=Jvyx<$Ba&>{$Pj z8wVb~^?UVw=fdXqXxA>{V4+LpHqTwx<9<-Wjplx9*$MBsS{-Iau($WKY8Uxt?DiKO z8g)L=q}`tr+8^Ig?8Q8LTkoqQ_RUkp*6>;Jlh1|KnP;3jYZ{Dg)ik%}l=zhB`ZuD7 zpK#c>HDkl$@yRwj*Pm(o zjF&ye4}Ru#@YRgNt9K3FS+VDq`;{844ZJxv=J@^{p%VsWeVY*Se(Td&Ssq^>N|-QY z{Idcv&+}j3v0&gY!-p(85jVM8pGMt$tIgXPUiWx2Ro8X7zw}!0@) zx{IznyYXzEUBP<|r(HaFC~4d4dT(8uUVL~Zan06NRn^+Xr_{~=e6jU(c*n|PSG+uA zuX(Y~xlK^j(hIwMdcVHGQ~T}-`G3+D8M-`Ub&YO*?!G@w?0dQHl8nhUF7$G_Ga~w@ zwXLhJb!l;IZPwLN`SW|NJ6B@KFWDUqWG!qT)Am-@6_jR?3?fzZMzbGI!n&ngvnu6^|Y8oqtCdTR*MM=(g4O{Tdn<wfywPxpBv_7>mzq~`FT4^EA;96CI$`sawU=gxN=vE-*! zC#&`@{kF`}V|8K|KDm|V@bJ#a?6*y4_bOE)_KzYTYfm`y%Ybn+hpOW4F8=e<)>Fg2 zJF_I_@3=vQj?|p|sb_ka-@x}7wws?ux3HPmu%COt$6w}-cTveJ#7)e4xb8sf5h(sbLSMXNeK39~6vw$DF8 z>HO0M?)9h|E-{*6F zw?ErEd$E6yE|ELG+0nCa=$bRjd>4QBecJ`6IvkGaexXi-F5bygy45JNJJO?0p&G8a zYr=$2{%f<4^ zyDODj(DvPmKZo=#8NAbNRMjKBqM!5*jc@*BL*pX-rk&U{&{rt(gKwAqE&dqm`%C1p zPK7_LTXnk5@PXb|MhSg{j@-b(8YW0+g!%AF!Q~ctC+XK2Z zxxZ@K+o#uV*AKJ58?&dif9IRsOS;d?j6T@sedYRIQ_HkpX^Sd!jb%l=jw|AQ#)B19 z#1Aj{JGGukiny_diZuow8e^V8v0L(bKIGYHvJ~gmGha->uLMEv$QQuJ5af=0dU)9j z`27CS;v;XT^lx*kjiz~SHTOi!-hx3-^_qV?F@zK z2aSr&21+o>g^7(%{*jl`-()EuOwVEB4T8U%16(rUg&_yt9BcijISx1(aP5=e!t3|3 zv5FOgf-uV4Mo@)c#I{-Yi*egLY6oqz7~AG%b52>NA!T+3SZEp&+r|=Qji4Cn61+s| zqGe9}4(EWdwrYrR5Va5kAu{I6Izp@tcTNnkIrA@CwTqZ#oNMT>VjJNdM!9UU<>8ljzpr8}TaT*-X?$jZY<;XQ z>dHt%&KdT5aFBUW*6&@d^!s+qpS5IFc#F^PYO%L&;=MJDL5nZN7Pr=04e9o| zGHs#x-m32rn_vzIIohhbiX}s>G3Mduj_}KzqlIJ~T?!n%i5y+Hoil8zAx8~+t37`$ zYEny-nd_~qgkrn|>aD;h=&c2%QW@yg7&aQNquBsno1uX!hN?|eZ?eUML(MtIav@Fj%S*x{KzZias?RBfS%F=xC z72aOejoa&qOOVEQEVS2$II;6e^TlGl40X6TGj6@TYB0eLHQ!)E^59-hIZ|AxkBTja zcNpc`j(wB7g||sZ<2Laf3vDt7+vNFO{#k#+HbIK1O_Wkx0GuAAm@9K$qT zE5ony{#3!Z&7ZCc(zNjOhef@0M|n{r4BI@vjxNSTgQKL){q=?$inP||TwF7N^fbtf zF;B-jlDv&GwUjYaTX<#$JS%}reNl>k_MMz`VyaToi-MVJ#fuUzcB~*5M|?)&_`Ew~ zg;+oTcK&|-M#hXDIT~Cf!edGlv0*}jRjd?b!YFSTA%FNa-WhiqH_WpV&@d_3Fq4XN zYHk?T&QXUG7DOb9t#w8;Or?($&C?)p@SOW|0P&s&@Vuy2V3Ii(Fb{H}BhP^S(grvo z288B@4VZ&e8$!cD1inK&$ff4}_0G0j1_V{|c9enH=VUUF0y2>8@BDS(i#}KuGjg_( zOH>P`T*6gVQ(TJX#bYgvO@$q^;oL#=H2wrQsO{YyscR7KfEP zP>yEHW~o|L%?x<)jlc7Ng4TwWN?U-OUm8vlq~Rn%k#7AJxs4&nt7k)mBX7mOK$r*- zTSyI&S_TdNe{xtEvt7leK$#fx^uSTcJ9!WM+PDY$l_AbOKf^z3C$=#teh@BfV{(G1 zjR~3?o{`0^gpC>hlgiS@rJG!B%`@&!FTM%V_zi+p|C|`HCO{XKH6lgD`T$WF^K{LQ z;eYV18DX+4n30*y9R)#cD8ke=vEqe!?L`%6Wkv=!;I97_y2Z(VX z%$tp5LtzFj3Ji%mi(ruu*%!~yI71mEP;jf8hroM!u7>+=BRX==o{f7`I_ce~PmP_x zzkoo-2k$W;r>mc~C`dnZLFy+Hq(PYhFL(Gm4?bcnNCK`a+(B5)Cy-+ z%nM4un5Xfpgx}(gKi|0VU+w@w+68(!=hP@(w|C|mzX*T*?;BsPm(@^S97L@Inbi3I zNxiIwvZ+yb9NEjTQ8gACH6m5TUcxsR^Js^^l6P^v`dHSeyD|e_ZS{AKjD4ZupY4}( zkQ!Af2Ya(mTo6c~5+r>}kYh8rkY2sC=~x0jZo6^e9fYV?)p!Ge#NCu$MX?S`S1}b3 zj8QH_>`nM>E%h0Q>L7LW4AVVLA!dDPZ8*-qqB-DdBcK)6Zj4dgUN~|i+D6dI> zhH6D%HCd-V^Fiy%@sO=8Dx&!!JX$EDChu3lS5&NGHINE-EY^6c$^Eb7-6$3P9vMqT z>!{3t?0A3Y*y=8g@hf*Dsle`ytQrg@Qma|>3vgPaEaZ>I;vkSz5G1J}DCW{%5sxXQ ztj{6dX{86evT#rMwC_eVJnEY!=-}O~*7sYT7qk+TKGOljNA00|XyOKGvH@{)hd&Ic z6EI9oD(+V!Bvl}IZt&~T%^su+1jStXE8?S3jQ(lfJF=Gpzz+-obAxh;f$(a3mtXI8 zFI8ZplQZvOeWf(P#>+#+4px3N}hfYJ9puAwjr-Z*1)eS4)x3SQo(UR-t1`DF@5HTe2D`R&qjCg%=%Q=DI z>UTNyQz1>Tf5$W_X9kB_xQZo6kIewV^(Wp8W4JsGhyEM|GLDk%1-;A@7GOPCeSk<& zEJ#2Mz)%*iE1UYg&EnmK9&Jt)%)y@Zi2&q`Xi=5Oea^|Ty#2>Oe;Wq4<2j%7diBlM zQ%gPuyacQDb<<#)AYmn_R1WUVUhRzE420eu0g^!Vv}@aD?h`1V;GhP%iEkiHknj*h zd`k3cu0cJA1D_+M0<;ob{cPeYhb^gsIoQu#$|z}5H6;0XgdUrL2c_rHu=qX<{z9OY zaS7fXq;V{~qk6j(8?z1Z6Wsgf)IyGsCMYjnX}xDo2;lBU1aKNH$^faL=Tu;PhX)rI zfjuz;+qR-!jTl9Jm_XlJfc5T(ay5vJVQ#Se7zG@U1i2powGU)<;TdYYVF0~A>wCgq zFz*Q-aCKvy;0w%!{lXX0fsk$iIea1Q2G{y?kPJ4f4CgP{kcez(2BJ zB9X*rRg+U;f9y;Z2!2d$vw|h33k2m=!dffoM-8R=;sy(BV;n4whHnY)=8PjV-T_{7 z&=g*gAEeBmBA-^yofzSkN5F~8IOJjyb^4+qu@+T6LdL9UXZ?hmDIO>ad)oQ z!U0z_U>7%;z~ECNZ|bfO6I!247YN?1IH38uAI}Q}FLtAPhITd?DFOY+cbm z3I?4T!RwC}8~YRnEx*DZr*4ar?JlGW1V6ei49s^CP)+HnGG(;4JG=l@V<23F@0GNzD=aN zR7SJ}w+|X}!|PS5K+rXJ+bv8JG(gF35ifZ#@^^;mVAsW*PZ3btcz7{idf8*(<6`AL zqzVKdoco8T31+)eDTqmIpX&6;0vym#3(ZZHO`4w8p@tp>EdJ7|L_7Otab;m`X^ zXWOO;1h-v#whPk)f|iTmav=H^sEJSmdFbSzR^R5A zkCaLi2o8HIrU{y&M@c_yk-LJohq!(d4KzdnNA&%W9caCOXduXcc4GgP71IPt>!1uV z@g7kW6u0@0zp&wD$5_5NEfZVxi~(K1nrLg zs~g``7Y2NH1KU#%>fH$9Af5lFmC8tpoA5MoK$<|%Co!u_hcRgaK`EOKU^_VLQs!?? ztW|S891XO@0A777phxM!XIJ>wiyc>;Xp|;Yeb=VLCZK~!X&scIK>W33&rBsR`)}iF zTeS0DQ1^O()f>2BXy5C?Gqmzv+)iLQq#qj;NG$C+EKMMIDP=296WrsSB%$5Gp&G78 zo3-%#cyZ1b9K}X){UDBK#Znk^%l)-&^6)f)VB7i~+hLlZ<|r3$KqF$)!d(k%Na6iG zyV4vl_6y`dG+>Y6y+ZW&(ev>DuOIaUdO98;FK)m!sqUaxVnMQoNw~CxvtAL~ zzi_-%w%j2gEn*N`Us4(gdFjx*~mXn?p5RmDr8n)`@xp z_2x*hUO4g$gKuZ@QW(2&{hlfZ@R@@hIMtY0-`;BJTT8$;sG8J2hC|N`0Xf6Do>*vc zQR;9MjFPZJD&&)@bBUy>X#&AQp)n;fO>ovFUL54s)6v2b0n)o5$J!DdmilQ!=#`$l zhobJneu{k-$0hQ9O6@hJf4ed2FL?pZ4bo~a$I}q>=foN& zxm-^Z%)zCBRFH^I4M`>aVT*(>1UMW8+5l}j1}(qBdWrqjoj05^1cC=Hr=??>pp;Dq z5W5mR=(c(cU#&uYjh=+)#h@k-1J(QC{1O%H(lP{sv-dSyG%GzrAh@=zxCqNw2LDhC zS0(jcxY?TZhh4`B#2a0mpwC;bnLg6_gBl~ zum(UCj>A0JZ7Sr}``PI;fDSWo<4{_(*7F1Bajb<(Qz~&zDx2>T{b9T+o9{w@k!H6% zrC#92Iz!DwuU0?qaY~GB=nD3R;H>ijZ0)f$fuOt|O8LFJPgPz&%ltad{%u6MKyc8Kx;GktJ|bue zuM)qh==gB%=%Q8}3H@;x$UN#El6P%0JFhdbsRF^@Smjy=N2dz}P2sH~e`qQlzk%yP zw8MDVU=+7fIJR)(e>kU&{SQZh7^0;zwA~-{{u=5X+c)9phWK=W(gdY!I)L(^WZWGw zr!6;Yc%%$5YNZlpIi&D;KY}YOUE^tjQZ^kxT~kVzGJo?-H}5xy`vsBE=2%~xS4O{; z}>SJbi!?}$Y<2{k$FL3!zj zUoC%VBJDIyl=p!!vlEU3XCRA&d?uYX$lqUH8Jz*EQpoXLpeejo=c@~^G zz_}B#mav;PZb28sOYrC1{{EOISfl_sKq7uEeA@z|wQ%1IesNyM6RfrQc^;hYF*km! zoWJ&;Ye%FB=HM=!bEcenEn9o0@FIG{Ley z#1TE-R`kdMJZf^D!40UOk-r<)AgD?3urjXcp-uCE85s2Kk&`sHk7$x4T=yI9@%R?j z=LueUA5j<61l^jaNjWn()WX$EdSVeMT(3t-M~YGB7_QeN9t?|q$t=?KmvaKaJ^R+~ z#xy}Om;Q=eRbmfpboAk>xi|(8$EuBh5>K5S-1DK?l6&*c3k18idG9uN>3M;m92MaN z5lD!>94yeWS)=a70KX}SI|xf$wYZ)64&Tl^+js|Ic6^W~^cDUNV>dg*fdR{!V7aWj zysTMy%Ze?6W%)ed1xaQ3UAK5Tnme{?bff|j$af!&y`?h&$(-;0GQMxg z40zpw^WEr7fWY9_4v!cfTUAOijf%P8&uMItiT5}Y%NjHBbqiqPJ78k2r?|domWiYe z{*z4H1r$$#hQXLect<54=1lA&W8ynt;&vrWR7$ZC6EDJ!!3-b!m4X0#tNv9DOx62l&Ec&?DPV_IWkBf##=ism`b69>=5cGCN{@QCjVM(D^i@UsIw zB;U0Yj0jqTJq!cy5XWUGw?b2kEI-0o-r1Ptqx~`i-n8|19(zN61mSy-Tf-B9O6zCD z@(Aw&md>*L&E1gtYK)=~qhjPIa_&T;c(j(cQ)}`gT~5X3TU@F*%n` z0y!3rOVE)&Ycfj5CFC4XdRzi!3d5J+(ZcuQaf!Vz99cKm5{Xf635vE#K8n&5Kfzd< z#!Sx)c&G7qn&2v|z^}?0ok7FtBb-FYfQ@)cd4k6!rbGT{EDi!m6G4(Df?_WH6{U&> zuKK5W??_7ArB{THj%5&}V;KbHB_TeQ{Go}oR(d8sF2VNgdfe)1s&4xWr3nf;#~Lw- z(?ZBsYvFgP7qo=Q34hj`Pd&7OkF@mRZ&kaDDWl(Z!=yC9&iN>#9xrJGdSeQnl<;G@9m zoQg07PfPf57#PX99uAb?WV}1H1a2G<1N9?4CkwIw!L3!Cog^ncoOM`T!gWILy zZwz45Zr;KD;BNO+ipPlxv$D;3r*P3Yfg60g5jN)wc_ z>i}X`BIgx~KI9`q+)j;4U9@puhxdAC&UYPNDXo>D4xx0T6mN>Ot<6x*u!v|XFuC&Y z*xpGG#QHKRt%HPNaCt?KpY#dgr!>KR3#d1X_|@<&O|1}Rrr#wB!&zrz+IPFXQeO0$v9G^M3dWNk;fWH1p{D?~uesMw-EgK%w z()T}?;9odig16Qk=4pZni;e58hNKexe$iyT#mVqk(6zXu&~<-%EBy3!f>$drQJA))=>bx_UYhoR>Qkh-np5=59yl_IKi(vj=3AtYCm9*{+i7rE_ zH{>tjCp-ju=B9Ca2nHVbcliI1@Z)$Ie~iGm&*`PBT&oKlAHjaN#WcYq$$9ZBkuIjQ z4us#Z_Ims)$6w_53I5@HiKhuho3H}TM3Mz4NceG!1j+$QfFTd&OWzV6f_GG5`L?75o^%uQGeXqn8z@X(<#&VOgI6i_l z#UAlA!FS@`ImoTYucZC+nZYmG9R29%w^0u{K7v!*KjvwI*G%A7(*F64@SEzDA_IsI zEw_vHCfLU1DW`*A*Vw#XyAnPWFrq_ld-d&LKjjI>NAOgDIEo|+AIlKMLu;g?${ef?>)MDXk>5kJ9mb@TFHN&O3&#ZQtb%0b*3 zPd;iP*mz7`o+kL$@VxkGky|Xm^tG^r-~6aW?2)t>50g;Z!mvI=T94{zZ3RoWSS^u( z-qiJ1j+WM}AM!Qp$;NAV?=(T0dA)FxQRTZ-BP{D=wxgw=pnb&4lKs~LENj7LTn{WU znwMpElMyUu@nEf%#0~j+AX4b|+R@S`qm$@6&M9H+J1*D^zT*;H>$SGYNGXL)q?-I2 zfVFT@6=dHg7yy3Y<<#a%;EzhS#Ph@g9zV`%;7)Rmk9TP5Y-TgGE931eT38n?Zq?9MPcfcF+M`Ljq{>9o5jZee^OkYV#+@;rqkJeNP(jEnZ^3qTjaE8fW z$kHdI(Py5$1Jc^6uGdIuTJyN!O9de&(Hjx(=(%ZlPFnjyzZ#ELia%d6Qa9Jg9(5Pj zx~R8GX?k9NDa|y09^Ug3Uh#M{{xBFGf7kz143&TTjkMM$!&_-NEpmMx-U||5ocqPK zJ#j|Q0IwuRI=r$q#t+~9yZT-e7dZTWsqCfA|sS{Nk#OSs7B@Nh4Z;C4r*uL1rJfY+6S3q; zZ7gEZlX}i&cAK%xe)JB~Bn|>k>eH$=NT>N9trq`}ops$no|K#eNAYirn#$&-2qu7*;cpeMB* z@<(HF5dMu+DeCC`m^>wF15@@;fPn7C;q zt={8!G!EWtDbMl}ho4M1t=omhq%^_$>nJ1EOdLOv&7?P`;AzQ|Fx1lSu=i2)8AmF{ z?RCG)^%Oyb9%9|+~-YKdqo^)Hprc>W8&@A|JuZ_sW)N)z;HMtMIEZ@PpRcR%6}qT!lUG!#X8Pwv60B{bIj7gHW=i{chN z*caS`t!Qkea_oaND}P4M>O*~w+N=j#Ng-v_8|79CJ=i3ma4(dDQEoHf0AeclU<(`D zjFmrw%?MTE!78QjOFh`0WeZyR_f#*rH_%kZ=g=U{4_?UU!1Gvl)Bm0d-&Dp61ZxuT zc^<4g|3A0$pc5{CS z(yVetPxh*)cFg|!m9Kzu4Ax|qJXtvhlzOscguVoi7U&c`*(1P{pJ6nLQLdGQb;;>m znw*TKX_aecz$fu^+I##;h$D@TDe`3Ds4i2=6ZB+nz_(}}c#!s^e~wfs@nlI#$de^> zWa$VWr3q3#f=+OwqJ#yg_nbwH#3#AsQ`)AcET6ZFMa!4XEuZX2_$N3em$U|?I;XTa z({0xBRh4RIP1?xv$)y@C-(sN9g5}HLme0{x#*$!1)u&TRET2*eztr+YSF&jNUUTiC zG1eZRP6cUJS4KYHvP!HiS>q!=MXMmV)k+)!k zA(*v%HTY}m(ngj~E^TP}z5xm?SiTF~^5r*{v9${`18v-uSU#l`eyQafS>58Q#XGJ& znj7=k#yv>0t{C$9Y*%EY`BjUW3ixyLt+VVLAYZgx_=gH#HX;__?tl*R^Hg37?!#5zDdm!Qp!bFH)V)5uyttE?B ztCmHJ_YarZb;dGl3-c7~VTPW4+TIQ6G;i@}B2K9#l5;?*#iKJxUxG&q6%j3-U+scS z1H%@K^0A((cJd{ZrWpyw(zM<_Gf?G$+9*2#zf$RD?=VGUy&W(ytCS~b@j@X#jrIPG zR4Ezjk(7|dBYd>)gCOnuAc*xc3`)m(CuI9Sv{E6~>bF{l3^9qb(Mo2uD@tdr@su;3 zE5|cfhCjU?CzWdre?e#qG3fDQyIKn_I{bLP1b1Z^pE0d7!YiN4 z6VE#9=hx>;MOdR_unv`;4jPZ^UYWS5WR?zbWdkW04K0HwtPF!SdKT-1Tck$~A3GQp zVCKHEL=T>c#NYBDtmft%l2A^Uy39Qm!ni}VQp zz^zy-8K2?5*KeAQp4ao6$RMFQNdTq{_luBbw8rPp^Sml@`;NU&-b3{}adDO*^uO+L z(SM$|v`?f#eTyE~74C7(m&wiS%s{*Sd{h{NJZ8`39EaxPK`F<{y)e!&9>^l9fhb52eW&?{J*GDd^tBqyVZhw$w^F;Kx zu88xHW@p#y0Oc60$tsQgO7&5suN?Wc2T zvNFe@lqcrq7(Pd;bVfLQ+n7$kQT7o)$sMM-Zi$yQuWQ43v9Z)azhiZ*+b| zX`7m|bRWGfTDlx=>53Ukwf*!U&6f8d)wzXQanj9NI!~!))})#&om{HX(gilMXz8wU zOJ`#&V_V*XjNMgY>6B9VrIzl##-gRm<=Vqn#^<|1nyvA`=bU}7k&)&totHvBf1agN zE3kCm@l<-^b2X`EZzSB4}Hxm literal 0 HcmV?d00001 diff --git a/testing/btest/scripts/base/protocols/krb/smb2_krb.test b/testing/btest/scripts/base/protocols/krb/smb2_krb.test new file mode 100644 index 0000000000..7aa78567b5 --- /dev/null +++ b/testing/btest/scripts/base/protocols/krb/smb2_krb.test @@ -0,0 +1,19 @@ +# This test verifies that given the proper keytab file, the +# Kerberos analyzer can open the AD ticket in the Negociate +# Protocol Request and find the user. + +# @TEST-COPY-FILE: ${TRACES}/krb/smb2_krb.keytab +# @TEST-EXEC: bro -b -C -r $TRACES/krb/smb2_krb.pcap %INPUT +# @TEST-EXEC: btest-diff .stdout + +redef KRB::keytab = "smb2_krb.keytab"; +global monitor_ports: set[port] = { 445/tcp, 139/tcp } &redef; + +event bro_init() &priority=5{ + Analyzer::register_for_ports(Analyzer::ANALYZER_SMB, monitor_ports); +} + +event krb_ap_request(c: connection, ticket: KRB::Ticket, opts: KRB::AP_Options){ + print ticket$authenticationinfo; +} + From 54ef1083bc0c61c60231521019164b9f24528f9a Mon Sep 17 00:00:00 2001 From: Julien Wallior Date: Wed, 9 May 2018 17:17:36 -0400 Subject: [PATCH 12/56] Make linking against libkrb5 optional based on whether it is installed on the system or not. --- CMakeLists.txt | 8 ++++++++ cmake | 2 +- src/analyzer/protocol/krb/CMakeLists.txt | 1 - src/analyzer/protocol/krb/KRB.cc | 6 ++++++ src/analyzer/protocol/krb/KRB.h | 4 ++++ 5 files changed, 19 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e1a9bbbe2e..fd9d599bde 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -169,6 +169,13 @@ endif () # if one specifies --with-openssl (which may be common). include_directories(BEFORE ${OPENSSL_INCLUDE_DIR}) +set(USE_KRB5 false) +find_package(LibKrb5) +if (LibKrb5_FOUND) + set(USE_KRB5 true) + list(APPEND OPTLIBS ${LibKrb5_LIBRARY}) +endif () + set(brodeps ${BinPAC_LIBRARY} ${PCAP_LIBRARY} @@ -286,6 +293,7 @@ message( "\n tcmalloc: ${USE_PERFTOOLS_TCMALLOC}" "\n debugging: ${USE_PERFTOOLS_DEBUG}" "\njemalloc: ${ENABLE_JEMALLOC}" + "\nkerberos: ${USE_KRB5}" "\n" "\n================================================================\n" ) diff --git a/cmake b/cmake index c8f07c6bf9..385edded96 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit c8f07c6bf90b472fb54b2ecbe0497238f40dec8a +Subproject commit 385edded96898154f7cb1d59f1d224cf47097531 diff --git a/src/analyzer/protocol/krb/CMakeLists.txt b/src/analyzer/protocol/krb/CMakeLists.txt index 95f6d672a1..1cac35d626 100644 --- a/src/analyzer/protocol/krb/CMakeLists.txt +++ b/src/analyzer/protocol/krb/CMakeLists.txt @@ -9,7 +9,6 @@ bro_plugin_cc(KRB.cc) bro_plugin_cc(KRB_TCP.cc) bro_plugin_bif(types.bif) bro_plugin_bif(events.bif) -bro_plugin_link_library(-lkrb5) bro_plugin_pac(krb.pac krb-protocol.pac krb-analyzer.pac krb-asn1.pac krb-defs.pac diff --git a/src/analyzer/protocol/krb/KRB.cc b/src/analyzer/protocol/krb/KRB.cc index f080efa7f0..7808d33893 100644 --- a/src/analyzer/protocol/krb/KRB.cc +++ b/src/analyzer/protocol/krb/KRB.cc @@ -12,6 +12,7 @@ KRB_Analyzer::KRB_Analyzer(Connection* conn) { interp = new binpac::KRB::KRB_Conn(this); +#ifdef USE_KRB5 const char* keytab_filename = BifConst::KRB::keytab->CheckString(); if (access(keytab_filename, R_OK) != 0) { @@ -34,11 +35,13 @@ KRB_Analyzer::KRB_Analyzer(Connection* conn) reporter->Warning("KRB: Couldn't resolve keytab (%s)", krb5_get_error_message(krb_context, retval)); } } +#endif } KRB_Analyzer::~KRB_Analyzer() { +#ifdef USE_KRB5 if (krb_available) { krb5_error_code retval = krb5_kt_close(krb_context, krb_keytab); @@ -48,6 +51,7 @@ KRB_Analyzer::~KRB_Analyzer() } krb5_free_context(krb_context); } +#endif delete interp; } @@ -74,6 +78,7 @@ void KRB_Analyzer::DeliverPacket(int len, const u_char* data, bool orig, StringVal* KRB_Analyzer::GetAuthenticationInfo(const BroString* principal, const BroString* ciphertext, const bro_uint_t enctype) { StringVal* ret = new StringVal("nouser"); +#ifdef USE_KRB5 if (!krb_available) { return ret; @@ -131,6 +136,7 @@ StringVal* KRB_Analyzer::GetAuthenticationInfo(const BroString* principal, const ret = new StringVal(cp); krb5_free_unparsed_name(krb_context, cp); +#endif return ret; diff --git a/src/analyzer/protocol/krb/KRB.h b/src/analyzer/protocol/krb/KRB.h index 52cf9fa2bb..3d924482de 100644 --- a/src/analyzer/protocol/krb/KRB.h +++ b/src/analyzer/protocol/krb/KRB.h @@ -5,7 +5,9 @@ #include "krb_pac.h" +#ifdef USE_KRB5 #include +#endif namespace analyzer { namespace krb { @@ -29,8 +31,10 @@ protected: binpac::KRB::KRB_Conn* interp; bool krb_available; +#ifdef USE_KRB5 krb5_context krb_context; krb5_keytab krb_keytab; +#endif }; } } // namespace analyzer::* From 8315aa36a61319fad352f6e663dfea1c8720cb4c Mon Sep 17 00:00:00 2001 From: Pierre LALET Date: Fri, 18 May 2018 17:39:48 +0200 Subject: [PATCH 13/56] Add bad ARP tests The capture file (btest/Traces/arp-leak.pcap) contains the exploitation of an ARP leak flaw against NetBSD 7.0.2 using Scapy etherleak() function. --- .../scripts.base.protocols.arp.bad/.stdout | 6 ++++++ testing/btest/Traces/arp-leak.pcap | Bin 0 -> 4372 bytes .../btest/scripts/base/protocols/arp/bad.test | 17 +++++++++++++++++ 3 files changed, 23 insertions(+) create mode 100644 testing/btest/Baseline/scripts.base.protocols.arp.bad/.stdout create mode 100644 testing/btest/Traces/arp-leak.pcap create mode 100644 testing/btest/scripts/base/protocols/arp/bad.test diff --git a/testing/btest/Baseline/scripts.base.protocols.arp.bad/.stdout b/testing/btest/Baseline/scripts.base.protocols.arp.bad/.stdout new file mode 100644 index 0000000000..4c6945ec64 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.arp.bad/.stdout @@ -0,0 +1,6 @@ +BAD, 192.168.123.1, 7e:8e:20:d8:23:a7, 192.168.123.2, 00:00:00:00:00:00, corrupt-arp-header (hrd=1, hln=255) +BAD, 192.168.123.2, 52:54:00:12:34:57, 192.168.123.1, 7e:8e:20:d8:23:a7, corrupt-arp-header (hrd=1, hln=255) +BAD, 192.168.123.1, 7e:8e:20:d8:23:a7, 192.168.123.2, 00:00:00:00:00:00, corrupt-arp-header (pro=2048, pln=255) +BAD, 192.168.123.2, 52:54:00:12:34:57, 192.168.123.1, 7e:8e:20:d8:23:a7, corrupt-arp-header (pro=2048, pln=255) +BAD, 192.168.123.1, 7e:8e:20:d8:23:a7, 192.168.123.2, 00:00:00:00:00:00, corrupt-arp-header (hrd=1, hln=255) +BAD, 192.168.123.2, 52:54:00:12:34:57, 192.168.123.1, 7e:8e:20:d8:23:a7, corrupt-arp-header (hrd=1, hln=255) diff --git a/testing/btest/Traces/arp-leak.pcap b/testing/btest/Traces/arp-leak.pcap new file mode 100644 index 0000000000000000000000000000000000000000..a49919258d084619d80e07a899dc79c7d9194ed2 GIT binary patch literal 4372 zcmca|c+)~A1{MYcU}0bca>76SjcS?C#UR7P0LK5ppsr8hhVpU_HU>rxhW|i?5I#ch z2xA>sQ4I>Kkqm%AzywMwM#~Us1(GU)LKuWh!r^I!3Brel6*rUyrYfRBuOXO?x#1r| zvA~gAj0_DBJ}`xX0diCvaM=fx{r~^MVW>Kwq;Dm{Dpmv&FU#RNOC!h}kRp)z3t!(8 z#HYc4L4bi7i-Hvlpnw|9E{w6jv{K;4#h?I;H4p{`?*ISDSh+Hqy?`kRly0yj zsF5NBN-s}55$Of9YyqYhXxRb^s)y5Kz#0GBObGq+&ABew^?fl63R!l@1&G3v2lgO7 ztmYGFzAFSOP$vBU#RL+cv)LAGjzJSd`~g&)>sKsToKX0)K^1&(6a=fcP+$NTU=A-J z_GX-e*rV?Nk;i79F#~JE2mRk*_rT>9)WPLkf*wTt0YV-UG$TCIBmvXQQ!91`Az;LU zFlPAzEX+sSEhE0&0?$t)k1@h)6i^AV)(KHUVAd$W5@NKqil-|5cYS13r6aFsfckiZ Gw;=$<%!O+J literal 0 HcmV?d00001 diff --git a/testing/btest/scripts/base/protocols/arp/bad.test b/testing/btest/scripts/base/protocols/arp/bad.test new file mode 100644 index 0000000000..efe9b1d15a --- /dev/null +++ b/testing/btest/scripts/base/protocols/arp/bad.test @@ -0,0 +1,17 @@ +# @TEST-EXEC: bro -r $TRACES/arp-leak.pcap %INPUT +# @TEST-EXEC: btest-diff .stdout + +event arp_request(mac_src: string, mac_dst: string, SPA: addr, SHA: string, TPA: addr, THA: string) + { + print "REQ", mac_src, mac_dst, SPA, SHA, TPA, THA; + } + +event arp_reply(mac_src: string, mac_dst: string, SPA: addr, SHA: string, TPA: addr, THA: string) + { + print "REP", mac_src, mac_dst, SPA, SHA, TPA, THA; + } + +event bad_arp(SPA: addr, SHA: string, TPA: addr, THA: string, explanation: string) + { + print "BAD", SPA, SHA, TPA, THA, explanation; + } From e35da5f592ca4f3bc088281da26b4c19db8018db Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 22 May 2018 16:27:07 -0500 Subject: [PATCH 14/56] Migrate NCP analyzer to use latest analyzer API It was possibly never updated for newer Analyzer API changes, as simply attaching the NCP analyzer to a connection would result in null pointer derefernces and also support analyzers were not attached. --- src/analyzer/protocol/ncp/NCP.cc | 23 +- src/analyzer/protocol/ncp/NCP.h | 1 + .../scripts.base.protocols.ncp.event/out | 468 ++++++++++++++++++ testing/btest/Traces/ncp.pcap | Bin 0 -> 66824 bytes .../scripts/base/protocols/ncp/event.bro | 20 + 5 files changed, 500 insertions(+), 12 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.protocols.ncp.event/out create mode 100644 testing/btest/Traces/ncp.pcap create mode 100644 testing/btest/scripts/base/protocols/ncp/event.bro diff --git a/src/analyzer/protocol/ncp/NCP.cc b/src/analyzer/protocol/ncp/NCP.cc index add7841908..f2745666dc 100644 --- a/src/analyzer/protocol/ncp/NCP.cc +++ b/src/analyzer/protocol/ncp/NCP.cc @@ -159,11 +159,7 @@ Contents_NCP_Analyzer::Contents_NCP_Analyzer(Connection* conn, bool orig, NCP_Se { session = arg_session; resync = true; - - tcp::TCP_Analyzer* tcp = static_cast(Parent())->TCP(); - if ( tcp ) - resync = (orig ? tcp->OrigState() : tcp->RespState()) != - tcp::TCP_ENDPOINT_ESTABLISHED; + resync_set = false; } Contents_NCP_Analyzer::~Contents_NCP_Analyzer() @@ -174,20 +170,23 @@ void Contents_NCP_Analyzer::DeliverStream(int len, const u_char* data, bool orig { tcp::TCP_SupportAnalyzer::DeliverStream(len, data, orig); - tcp::TCP_Analyzer* tcp = static_cast(Parent())->TCP(); + auto tcp = static_cast(Parent())->TCP(); + + if ( ! resync_set ) + { + resync_set = true; + resync = (IsOrig() ? tcp->OrigState() : tcp->RespState()) != + tcp::TCP_ENDPOINT_ESTABLISHED; + } if ( tcp && tcp->HadGap(orig) ) return; - DEBUG_MSG("NCP deliver: len = %d resync = %d buffer.empty = %d\n", - len, resync, buffer.empty()); - if ( buffer.empty() && resync ) { // Assume NCP frames align with packet boundary. if ( (IsOrig() && len < 22) || (! IsOrig() && len < 16) ) { // ignore small fragmeents - DEBUG_MSG("NCP discard small pieces: %d\n", len); return; } @@ -224,13 +223,13 @@ NCP_Analyzer::NCP_Analyzer(Connection* conn) { session = new NCP_Session(this); o_ncp = new Contents_NCP_Analyzer(conn, true, session); + AddSupportAnalyzer(o_ncp); r_ncp = new Contents_NCP_Analyzer(conn, false, session); + AddSupportAnalyzer(r_ncp); } NCP_Analyzer::~NCP_Analyzer() { delete session; - delete o_ncp; - delete r_ncp; } diff --git a/src/analyzer/protocol/ncp/NCP.h b/src/analyzer/protocol/ncp/NCP.h index 713eca756d..f8cac95090 100644 --- a/src/analyzer/protocol/ncp/NCP.h +++ b/src/analyzer/protocol/ncp/NCP.h @@ -97,6 +97,7 @@ protected: // Re-sync for partial connections (or after a content gap). bool resync; + bool resync_set; }; class NCP_Analyzer : public tcp::TCP_ApplicationAnalyzer { diff --git a/testing/btest/Baseline/scripts.base.protocols.ncp.event/out b/testing/btest/Baseline/scripts.base.protocols.ncp.event/out new file mode 100644 index 0000000000..6374c60f5d --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.ncp.event/out @@ -0,0 +1,468 @@ +ncp reply, 13107, 70, 0, 0, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 86, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 79, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 86, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 79, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 86, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 79, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 86, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 79, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 86, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 79, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 86, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 59, 89 +ncp reply, 13107, 2, 8738, 89, 255 +ncp request, 8738, 59, 89 +ncp reply, 13107, 2, 8738, 89, 255 +ncp request, 8738, 79, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 86, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 66, 89 +ncp reply, 13107, 92, 8738, 89, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 163, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 163, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 163, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 163, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 163, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 163, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 163, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 163, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 163, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 163, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 163, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 163, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 163, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 163, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 163, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 163, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 163, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 163, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 163, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 163, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 163, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 163, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 163, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 163, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 163, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 163, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 163, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 163, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 163, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 163, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 163, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 163, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 163, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 163, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 163, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 163, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 66, 89 +ncp reply, 13107, 92, 8738, 89, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 163, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 163, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 163, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 163, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 163, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 163, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 163, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 163, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 163, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 163, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 163, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 163, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 46, 89 +ncp reply, 13107, 88, 8738, 89, 0 +ncp request, 8738, 40, 89 +ncp reply, 13107, 11, 8738, 89, 0 +ncp request, 8738, 40, 89 +ncp reply, 13107, 102, 8738, 89, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 6, 22338 +ncp reply, 13107, 10, 8738, 22338, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 163, 8738, 72, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 163, 8738, 72, 0 +ncp request, 8738, 14, 72 diff --git a/testing/btest/Traces/ncp.pcap b/testing/btest/Traces/ncp.pcap new file mode 100644 index 0000000000000000000000000000000000000000..d8c61b3683f631fa404980393915cd9d97d46da9 GIT binary patch literal 66824 zcmdVD2Y3`!`~E)(Ed+uHBB2^qx}>o)BFH2HDPD%{^wmo*F+t^yCTMWXG5Sfv}Hf9?1|D~a{2w%}& zO@k>qra|u#l?JEKS}-)o%=%E#AjRFCTEG^S5Ma(W7n>t<`{w1Bzh-9}S(Bv_v?*3n@9aLK|8iY9IF_tXU>oo)$;O*(ees0rhhhEfoIKzlVYd|qZ^ z^!o5wF~&MVj1-6wUbsUMqp`a?g_gTn4x-YrpuHM+kY_C}cnZMv0H3n(I|aNg{=O1? zDWd!1OF}1dyI)1*?HRdVO|A#|XplP>tf)#(v1RhOs|}^acujjX@~NJ+x#Xik9uD&1 zi_rNhyP+9>Pvn6*-7-Xvcc{IxqqL=&G?oHeNmKxpsY$hMr0AH~fWU~D=mt`iKGRUD z+9euDDdwWA;=FvPjNA++z;B{j?!W}-sjB6%;Sk`8!9Mmww|j~JsAWQwq@qz5FIl3N z=YmX7%XPRSUV(@!A!28KEiVTl)iN=XbaE{_{w}KJj!c6ds#+eq(pWLR0W?^b`j?^s zYMENV_CKoS+J6XZxh_wW@eT9|DW=D4zUVNXxDNlCPsHvMU1Nh zF*-wxNjcvtVxX2Ov|JJIAgZ92>v6$51H2i)ck^p`C8AR;6FQL_u8GJyGjd-wc{7j) zfjlewgR0~dTPDxFF05r6E_o2hJAix*zm``a`v0Jo!*7UcxeF7(M^($?IzWKU5TI@y zKSco4G9gM*y_AygEm6yRK_;l>`dkq=L&S23XyDiKY7kN_6C+S7YdP+wsFu4j4ZKyg zJgyuXd}^?dxiq+?q5*1|TEO-{s^wiEjYfFdtG!7CdfIX|`4pObXRyEj2fvorfY_vC zM~x~X>Mv16yu!rrQdh)x5aSPsaWVs3l>;@@GKH2a;>aywEjQqT{{ir80KdSm<+X@T zwM^(lo^@M9-i?uas>!bzD<;^2e0|!Ds^k<~Chz&Tu$CM0$S2r>{2!2i#INObi2gsQ z<=a5j9%DN)0X)>T{0{`kg8=@`1}OrdmI+akK98FJz!J5*{*JJg?YJW5=@Z5qAz~T7 zme+%jYMB^;T3O4HcSW`A#58bM)$({FG?)zy+P;SVRM~w|%hUq4|4}WU0BJPB)6S{o zMqEvBRy6)?Xz~KTmN$Udq+>^oDkANks3M%17;dVH82>iJ*a0!DQaUSQpq44LToI!Y zRq($292a~Cz)u3aHoulPB0AMFp%Z!Q`yz4|M((O6KMC?r4EE8NlkKaLQ*4=B|4>-V zjk)BXfc$HN{k_}#THb`{|ASh-3smh|c4Y#%sA_rq*T#y8?hs()&&?D8P|JiUNpnAW z{HG;qIpLA8mY?T|IMH37Frf=XT*0s9%^;*&CPtuE)^hECM78Y3G;mhc@`NtXzyu8j z#$_rRpq8lxZ2zNLZu?kR%T2hNn4rm2Xws8k%UeKf(y^mP74a^b&>myEGclZ06)|Bd z#3+Us`b12fJbhoL&~in*@tq|a@ z!9MCh(|U>ksAWQwq?PTDerkzYF0O4S8e=!(ig;I_Fwx3jfA<4^EpG=Q)iN;xwX&9N z>(~kAzPy+Quc&HyqLr~?QY&aMFM5`u0cx3A!1h0?_=3=`*L$GxDUXG z06d0Y%R3RBYMIc9Jhq;Q+=r2OQIlgDchUfmC&Z^$C8yXj`P%k&(geJwy_)3Zp7vbw z0U#d-@_*99o$)zYWkBaMff9M^4sZnb4(Ma+AG>WY;3u_I#J#P02dcw)Cv#>&rzh-Gd3xQRUJT z*BUD(zXkzfN=_>Rz@-UMk_y(1n`sG`egZPVOh_x9h?8G~i1!WlcRu60^xGf|*&s7g zX~?sh8&`;2x+l}1gW9FV@Hj=?kt+n9&zcJ80}RRG3hmku?%7~jUJ_l0hgxGve*5Cr~;R6!v$Xk@Em}@ z%6I9#h)ym|=tMqem596-BY#;MQ?P?;c9iA^$A*yASZxb`xYs;?}p6g{e)Ee&c`r?j~ z)UwIo_^IytJ!{(}77cZMJ}1A4KJkqSpZMQcQ0Lk(=Z-_uXYIYyFf%Z*!K@Ds-ghqc zov>$}ar4NWm=iJnd9K?YbXZ=i&-fAR&+WYHHY{LO$G-|LPl`I&=5gMsq3f)^PImDq9=NmJRaHq^3Bg(Z_mhz3(x;@>!}u3 z_Zbc^*f*zQ_xuGRC-gtqIIKOIows6Y&$I(wd@h!IS$|V>VbH^OAK%( z%f4%W=bStJq-;xZ?ipV?n%mQCl%DZ~ZIaz=bG=>W)t1Jb8Ms)2uypmYoD_gvHE;J@7ZgglSUXT-h2V|xin>` zQlFXb?nEVT4wBSIl9pF~B}rRH`>>1P|Nd9sL-~=kby*L#TR;7l@~WZa7Zua5Z`epz zr`V`4;%z?HMw)}4(_T%@@t*B?YQFgbDjyg89?n~Yi@=rFCNagx24T21xu4V&yo+m% z(zG=F42;?nCX1P-?~PMwdIg#mLet(4URE^4+c&XEQt$c$pIBKrK-I=oRK*XlJ=OHz zS5+0QrTjjr*JqxM=P9{?e}eG?<`G-eLne!h)#1z*u#tzVs8 z!ZbM9`B@Dn_kafTpg}8}4-^e{y1P>g*p?^s%Plq+=9-J6)AI_8lZO=JCIfT}eR^?m zes^c*($Z3=;*1u#BQ&y`62sFc%jWH&+io7e~CP{HO1~ujpGYE*@SZ(7d5AFW+77>f6oD&Xxq8 zhP0!>N_(~L8R+>kSKxUN7?&&_%;b*+4kCS%j&%WQ!hL~_U@WkdDf{jzm9l?8*|!Y# zk&nY}D#~IkK%wRL!bn6Fj0M_r!QTS-BY;=%#{!2C-5>D@oyhM9$(J$m6Ke8DApacX zKZahcN=~t5^0Nztj?jTi{<%Kk&4wT^;5))$M7N|POj;yzgyk&r<0?mZ6F2uvaYp8& zKOL-O4o4t1l+@$5)>*<4CNCB`LPwr@Q=FmR>rk%^-w}?0kQ{*+fm-PZ9!o@yu!3oD zOyvk~z77qdp~0Y9s}&942-E_$n&1dakajd!Y3DdXC$7NJ`h+RZ;}V6&*}*$V-=t$* zfa^s^$Y$;Q5KFsq2;=YM^u3$bmoHR1H2u;HO>x>B04z& zp%eKPA^9ps{*Icw9muDF{Np(AwRl1~G zHOu^n$`PixB6GY^M4W7o;l>*WMP4$AZj!<^2|9VAPI0A*1>uM9C z3LK#u7u*rxxI^ilH-C)qE~1ko5IT{2tP+v0W8??bTPFV$QR$3< z_G*%k_jKfvhk_h)#rHb#9pNORThbAVR*M{AJbDwi2}d{%GJzvFan;)h^)_K{<4e9Hyaz&Z1Y)FvBA+#+uMs)I2ByJ&l_N~u1Pwkm z*oXggA7{vwt_Vk<7O>R>M>veMqrpmhweA_{>C6@QV|~K31PH9-JHq=&-=t$*fa)r8 zt*EXxGG+Iv>S|g7l>H6LzONsxC<{lR&~jbPL{xzzxNyOL19%p|oA4ds14JiBAao*k zSSKRi#K`xm$+L_V(`-P#v1egba*8dJ_g*h_1XmvUG#ii)1GzQd5l$hxB^}`k7`5jS zHnYt4s2pM1Fl3&J%tQ0OQ8I@k5St{OY|(#%B^)7igU}J&xay@sJ=~^vXB*!UJ_I2- z0x<%$(h+__OWN}YTbKrKs~ln4r_f+BH0X5OUC{uJKrLXa369WfqtFrDxdKntCrls3 zR)SnmrnxZTC0}dQ=}sekla6%(s;ghoL=b*Td$k!FpRG*U-Kx5pJ_^d>l47{6&ufaZ za0Ci1*Hx`eLPzl6g5#3n^ck38y1{pZj}V<4fzXM38rsk%-^R$_Qj^aB`ALI)*dN~E zRmmx~OzyZ@=m?%%@{=H60rDUDj_@&}Thb9Of>GNMwzJH4sT^VY3S-4|EL{ql_M5Yk zIUIr5B*`?y=Mzgfg2xu2BY5%Dn~tSR({Dq)DSSux1cc-W#0b<%NB9&iX*i>Zk?-=Q*4=h{dS=v_;Sgy4r&H`=D|U}BYckNmUM*3 z9U@0~i)Fq|&_Mx92zNcgkM<6yyy1wxJgO+fFtsoPeHFW2y_oF^xW(%m7 z#&?7-KuC^2j6kh)gy5YbN7&6Y*s5}bnJtVJGipPFtyixq8o&{#1#C6J5q2Q$Xt2`G zIcw;_6L>~#2>c^Pp@598`pK-%QD||_Sx%0Gk-(oOAU<^`aJ#} zsj`h!oR3)A}MKmP`S&jUeNQH!9Jn&R|^yk;4ahxwwmBBe<2!vKdvYX zccIX7jc!9!fxGnKf`1S2Cji&DlKCy7le-W)k$2xKB0tE;*Qv>$fE-s#2VQKxt13Cg zmdVc}>KweLy_)3XJ^i@kkMs$%p98tZmCUnwn>i5bB||-pE1Bm&NRB{^bWmhRDBmw~gu_gO zH7ZA#oeT{|LW9Ilixdsu2-E_$n&1fS4hS8gFIV7^`h+>SRp_q9mCW-<-=t$*fa>a9 zG@(6?Kf;t|A>Jp*~93g-Uz8K)`0j_Z+^8%uiBM>@~ z4?id(e}|E;Qj@m_`F@bMZ!)ziImMRA`yLWHLLirXKgc~mu5l&vJ4CmnBRmA7_88$P z%Y3EE5$1RpD`qt}*vG%OVz`ny9D&#*>FdNFzOjTO?% zgyaas2-HePXmLd32*;QPD^!j!XCgGfeO2+RK8#Q_fFn=~*lL0!Y(m=6V5Pm<8&;ra zFjwFn`h>Z@+Ow2DAbpdLbpfiY{_lwD>Nrz&xvH+_`a;<_C_Apv(-9~ffkMl5m4>JS zM+o79#{s;b_AKR(h)#|`=tTaPko*KAU#2GS2l5h-r`e?|eFEeZTP8nvROkr(xa1`u zkJp~1yol(QbcE7lB1d?aWxiD92y^3+`B%ui{(GC1%;5;cCQ0#KUG`bR5%kA}j?ka0 z-d9i$EA_NyDKCML9Dx{tTImRv(USHoT&5|ty&#X`GTSi=(c^T{rX25jJ?4bq;ayo~fsI@SfKu6{rh+GB+Gn6isibv190v0`>JDEmSEXhm7n z6@`}T>H(q(93hMcKD!yfbG2tFe?oL}1VShBdGCtI-)H2D)a1D!$3p73O*Vb2l2dG% zeCA1^BZPCwu?Tt|Mx$D@ls_Z7B^@F7J&_}Pz%pN`a)fy!k@-wy9@hT8k~tiK*d(d> zAC1OZ!Vy-3OmNl^!BuZ2)cZnvmhu-6k|Pi!P%9lF;C+!JoMIX*P&vZ9FQ5VD!Q(m~ zt*vMPN1zt4)dWXahO|+0T4RJruE1~W6XuW6o~68k^i4X}1*ooiejuu=51F#_RdqFg z43xz&jB8NaNl_M#K%wQjibGU^BN({gUjTfz_AKSEh)#|`=tO=~NPe1;&r_4n2Kfzx zz3E!5wpGa~woHEUL!l!eYW2qm&b|Ti)!MU^R}tNkjy!~eIUmO)#UQey{hCCTP9zHsB`d|_G%|;<2_@!qy_EV_ksis^kk%U46lnovx~@g&#xNFep3o{$)j3I0A*1 z>uL$23LGJh3myjW@3dzYZy-830-+Omr!Ph1Uo!G(YVz+uo(b}zdtX*1r`R(2D_;p6 zA)ZT~3G!>&GmAG7-I9**BN(-37QbSdPgOa>!fVJJt~H>U!K7pkM<6y_9kzP3!V-?q z_#2@k4CJc!fj(i8pZ3h+Umzq$AV#28I>HIGq&-IXnrSdaSGmE#7KJIOxEB!@b&CGmCdXNRB{^K&^Cy{b)(s5zaCVCaN4^ zaVj*xI+0k9%}<}G0Y{(~u+;=dXm&y92#H*Qu?S@eE;DJ}|9BVan{=!TP+hG@6WWe& zjww4qRaZ-zL)rJBZ1d|2mEHwMpwMz%oj_E<7~yp;_HzRmmx~OnwDX1&)x!CI1QJ-r6&Z_YvKaj_}?OB1gEuG9RaM zgeBg_iuwM?eDU;bC3B1sh)t3%jyQVK5{}UPN1-DO=BYQ|AL_lSJ+t@#gyaas2-HeP z*nyU`9pO8s!B~|eEO`?ezx9pQVX>=;#DE$sqvW z?X!j-S>_cgM_BqQGROU2(ch21p=1t6AT~+*&h~@pmT&~OpM;K(%2n?P)Z3vwv-lW< z+wfXCVWJ5GWlPz%^H&k`CED6f?fwD_~j#QL|BT#6$uD(Q6fg_}I z!TSKbKzn9ULUeKjLMQT9e-V*iX5=H)LcpMUL<@(_pyD5tiX0-V3l!Bl(3t3pS};tGsKD9gud&n(tL`X(Lg0#sMup$TnA_=PDuOjTFQ$3xkFpzM!7zM?1# zN1)JhT|Gopfg@yd!T$j`o}Z<4&th#vCr2Q3A|L;oi2Mp8AF3vwXRKJ*3gpLsXjPS* zV$0-_zY859hey5;4=rB4PJ3pt4x(Gq5$=OgdyMca%e-9W2+P+Y^KxXK646)59F9P2 zlGMX)!A(mzLf#)jN66)>R}S?aXwNLx1tB>CF#@&H5$azPIl@(@L7B=CmOp?7OQAuZ zdEY4-z!9hgY&F3VHY4q5u+q-C){w^)7z;;MY}TGxtcUbXI@SfKuKfNK)zxoI*-}+q zt=J4@4?x+M&(~Fyg(FaCxvmlsRp1EuT<`+`Kd3#kXoKkF2!u}Le+bEcXXGVn@`E7% z7UXr#-l|oHkZ0@-j3w zuD@Z6RM|!IA_ zeZtC@v}Y-8K}haGj3j}4C3DSRB6sXHg4i(6 zbMV7WmT-hxcZ7~m!d35TDX9xkU9Gz-s;gT} z*<4j!t@<0vVwI3#<7bByW#I@ETCS_Bh$?V|GA_7>K4Eno?ODpkh)#|`=tMr|o{0Q5 zBhOKhudV~~D3HJT>6WVG6k8@gb6@BP641cc-W#0b<%N0{?a z z(20E9KO*vbj66e4-ql#K1Z$upKlyM%RdR|glMj3>bc7K+@+HrKyu0=+Wiv##q$4!J zZKv9E68Bl==_*H9(;b=TAoKjl9!lnL1Y(n<^4$SPEHOqX2ARMSMsn55fqHmKuh#vK zFM^O9ff#{W=?IQeeZd&v0n;E&`^#zXbkSUw0s;jlVq3kv&>-^p`Pf?`Ma$Wf$ zs=yIOaly9%JW_j>(jL*t5eS{gzY>x^V&rBuc_hd`0{QbN2P=I7UPiHH^20~!t3To6 z`M%x0?pWm$KG@f`PNQ@1C)jMA!fTZAQfB{K7tB~+B2RlKx#>MIrM9N zfxG;}R7_F1%i2U^#Zq{8#9J5JD=NZWhz(EUTb5*z`y>ok>x!1a~R@P3SITxB_EA z%eoZp8P8ToAJ3AcERfN}=T%bwgBvyNu<6 zX92tzvsD_;l5CCW-ip8t8L00V-ja}vZRNfrK$-?zkgV=l-9 zyHu0W|oc^ogPY9D!QE zRudfIDAJAwEA5=|#sr?g%dntjJvzPCy_7E@eUpxL0jewWb&(^~V#*Fu)z$h*P_{Fa zefv;nMOiokg_i4TJ)#O6VImj2Gr$*W&r&)dIynNN6S@9|h`crZY5=))1H1#C6J5w_hDI>MV=fw7=v15OXLo+bG*(l_Z?7ofT_+!obUJ*MnH zRb6e^24#Q7Gg|NMg_M==H*f?BE!R~pq6!>g3K#rmfFIGGrEHJrdt?>Uu6zPuL5Khd70?11Q&bcCEcB1fpt zGLKU^!iG(y=Z;bv5XosID3?We2G0YU7tsb`_NEvGaRHSvUfPmg{OLq6!>g1{Zu4z%Ogh zQg%jkas)yr^7{8h}SQntW>iLhTuAXDcMycv*(>*AQnY7T6TlOo;!VxI6TvsWG zDsY53T<}DF!e$%oS;}sRPL4q6M1EgL-k6aaRpgs(Kt2NGuW#O1m7HSBTv^Z~A$bxj}X3)Mk5Rex9B;{q-A2m2ITryNIQI2GWGZ zid4BvWh^{z+T(XcMYs#)DM=aDeb!jQUAkD=iri%$_eh+_k;vAbrE~%zxeGCp1oC*} z3$&zt2GW#i5TSCH&DqfN2|aH*+ELK}?m{hKs|oJnVQniKZ_MWkjKwiq+Gx*GIwO6P zj&%X*T_2+f?W>|MFlED4HM*sZv0^2j<1z5b`~ixxa2EJMelLRtS=u8Oj71PU$Jl|P~i z9AOC;{4Bt6SCiJglpcsqjzH){enCjyf{}-)$)iAi7v$|{wyR1`v1RhNY=n-mluLdW zpSg|S=PnUf!tw6~fjzDaZwD#tEKUu;N4uMQ? z2C|H&-l|xrw?})H(hG#-2*e1~N=F!GD{_QZOoJemBW&FR4X}PCepSe2MFThjwScWA zIKo#*TQDcFoGUOE$81Z}o~86g`X(Lg0#sM!4MdL6nkgHos;g~jP!{X1;>S&Gs3;3Z zpwMz%%|ldyBdp+p&jUE_+16U|?Sts#2!u}L?hQrcZ5Vlgn!FU`yFs4z=A)|Q6k8_0 zfT(luns&}CIWRdaIj2z4O|$l-)r{jzElbP~@|Qz~@Ac@DkI&U*!nf&O-w) zJOlOTzDE=d;0V+LwwmAwJCU})5ms{r#)6jZQ?zF(dmw$2j&%X5tMJC6x^iI3`l;$_ z`xGb}4P`%=fPvf7u{VX5>na0L1&*+W3my&dCEBx;JrSK8fzXNE_IVL`J4W6|O}+%= zSXCakY5bY0$vKDs!!MZ(0c zR@g*uVo{(-`2#v}Y;%AUZh$p%eLGA$bQz z-cwC}4CGFDy6m-zT~)~`woE?eMWG{Xjyq?bj&3NnOunY2&=IzB$$tm=9qpOLKt#8sBSf_lIYL*K zxwpy@cHS{otnG)V%YO0F({UXff!HMJXX(IcOE|(>kO>@N8&AEp{qzaD`fJZD27!*(y#M#!e z%C~hIb9uCvO-wJtq1Ib|{cgUY@uTbYuObZFh|n>xp@go?_G)tyE}r|Vs8Gx5)rMzY zn$i4+x7yI3=UYFL?(~-=>)Lg8TUph%rf7SNRiZUGAD-Dmj%gzWN|MX-@e;^pJ^|UQ zME25*hG*7Zy8Z$pw#{$y;KtynXQSOfG%vJ7do{Mso(H&WZ4t2<@{|_hQU@_PM>mvy zt0S>p9i`3%&x7J9%OQ>-GCIZ~y|_5PyR&mnQbx8@T3%jSw%I8;FNb~YoSdGNn`_Q? z9-NVz(tAK)l-{djMzP+*&BNQJV^XocEZ05L-L-Uxhfh{|VP4iypX{N&8O5$%X0w~O zYe~oGegS&_pa3uTKsVpMo^HPG-a&pHqwq~{7dJ1rjzi1zse_Z!GQBf>Ts=z*Iu`3) z9YSJa49>1ju6DK|c}2zD9aL0ywy{O#LPx(eb1r@uo;NfjJ3GnQ)5*o5vk8cKrA0j) zVtY7LzIKR+aqx6W$)M}gkO z$+ufq=dSFP&%c2j!;*5-h9sq#yAw>>;BMwz_6wfMF9hZ$=cQ!irge8n8=8^dEybLg zom6aAe%v=bsj$clpnlPjjy^t~zK*Wlh7_kd`miwG-Q@PX?QBEM=6pxL?2HmL_m=@?UA{PGyq?(5YXLQheUW6 z#U~Y(FsHWhuNZy%|K@58>wjKK@N%H6YvLvXlX zzpzBVnE%5CdN@Su*&pjm6aCZj!ZJ*z^whkJ0sTV;`j@#m8hbbxq5^~TaD^ma-@zVk zDaoGR?r!cWNvYjq^`3*hl2d(BlRQ#VQ@oPBeY(f^>pi>2gzH`1JU!gpJYC(qx`#de z%EzsHzyQ6UU-w{quY0sUD=#~#Fez+ke27P|Ii@hbheHI~ar4$^1iO_628a8`q-RIO znj$k}W1=JbC8qgiL>6a6c^9}8N4ZBv=KJ_0<|M}#gtJGs?scWHMu48JZ>%j7`P_Nj2CYP+> zvVH?xlOjvvV@u1zAqqg`{uzk=>1P};IATCrsv#hJU~ZVZTdsF(SZri}9~Wb}duT@g z*vRzwGM|7lcNd>fa{fX%|A$r*+tto*b;$D&&-wR#!QMaYA4!YbN>YKV+0EB0Fs?kw zFWD4RX7C6P_YTO6^z+RM@C+>V?;jQ#?dVusU~n;e#fBCP7@9G_H`(9SoYg3(b1fi zXDkZy@Nw%~nCxZDFK{o5O!V{*2p-~5Fr;5VT6C&gxR-y4PhVqlV)5W~^MJmwp(WXY znbF04`weh0gt~cn`Q^K1XQdd24)sn>^3FATgv5K7q{n%?=HzA+IYx#L88WbcrbkhK z$MlHI0j4BVzp(IrzG*Q=lS@)@Tu@w*Pe|;5(A2n~Sg!!Q#TBLXa0sD8;Flj?oEaP( zWQ_LCNlx^Qgu%rO)H`}H-%LWk?e379=a`(7oK9Xz{~;$WF3iiW`U5-Ln5H zXfAL`EQXirGb&$a!0hmb>ES7VZ7yLy&B+^5WX{PeG0WdgEK1fV(~;fsdPBho==ni3 z0*bO$jWe1x|KUKYq49$o6L8O{yiu&)?>CgjTT5(L-*aVM*2|7!>8^7}Q}&Op5Byb< zS~eLRKh<5oXKkCrqM@$O=j1ohC%!S^6aO0v>RcP<+;Ql^2Qyb+jc?hg{)I8EV%o1> zx*|K@bJ#eqzveU=fAr!z>#rW&u;+`}d(t1Y9Xc#%kB|4^h|q}d9DbSpLAQBdgtYrU zXv!~}_BCJnL$s@*XIcD&vFm1hoRiQrVX57>13u|rv?KSC%}eobct(HIY0r}EM;YDv zy}8QV=AGi;dXB$b)ouAGw^=WrFXt`!xXqF?qrWWQIc>p>pZ@Ce#y7_nzmxOgb7lJ2 zmM3R;mK2QY+UTTD`nuq!DEgzfO_)eE^PNdxY%_`VCy6J|% z`i>dLJ&x~4?>Fbog0tN=_@x}Ny72lo`*!c2c)0wSYuh*0jU7MnSlaBF``%4E?QHJZ z`=0lUsSVw;Pwwhwqb;s;Aj`!+c@6ff?(qj#`=l0(@BFyizX_wr)KWeh!+OkHm4|W(^|1$S# z$Ii22+Wc7eWxbu-HzT{eDz)6u^5dMQKkq(o*C1=)TI0rMgGojKf2%0ySE{R!KshsS$WS((kg#>(t^az)rWHEl1y`p^F*{@34bB;x<1 z3O~+MY+^=x(fLU)!;x$@c3&n*FC)tOS+$fpujFr9IA?4?6@K>u1XK0=cI9l<+bZ0!&kYG*qj119QF(ww|I)KVT{f`K&rZt7eLALP18Pm5??!J0Ma6lAXeB49%n_q`y$fCrDJpg> zG?!pNox(=!hC=llN3^*ZZyWNs^#2?UB^9UZ+1Zz!Z5(^E`FD8k7@1Gwcc_t@SCpHP zno7SA7ai>g`HKG?+vAPv-#ni^qp-7$HWy;ZjgfIsAWpH|?QHFAUwRYwePOeH;&yS9 zNB?M?O`@D?N#5NrIo8E%>JqomC3@mD?dmQeb#8K(x`ZL)*)DP85_?N}{=tp$^jB=7 zSdClN78TE@&lFGPN%?`)wU=&OM)AATi7!QQZ7q!Qg&@lFPotdbAk6Cl%PUot*Jb3D zMsKy_u&noKUKvN~C>lMrqx6!iZ4WvuuhnP#i1p`o-gO%mu&U!<1(zp99c=SB@6^!u7ySPF=#|ah>%61I;2HNm z+vz#hZpRdQ?Yyg zf{+vXA8Z`f9?i~MF|}vffi6B5%e}0>DY`J|;k%D-?wW9Ft$9yrev>y^{~UBF_`Q}+ zJNyz`zc>69+j)PEnc;up_sv)1=bY>J>!7hiPr6IqkwNQS4t&_J(Cu7eROp(R&%RCc zntO0kQ-`CITmO-Adh!wL&RtLK&3ZRv!O6t~UU)F<*pC}u>aqCl+5W@ZM0MDGH!pA0 z_{ph9R^1)jD*4k(10BEfpR?xIjLl8XELr{CxW9M5cs|T-)VXZ|E1GS0^3F}{aqFW| zOFZW^?YZvB)}Q?sWK9a1S8v_O!r|Xm_rdG5-uBx&ntpB`T0p8i+gL-~=kby*L#%UL}AKD}xv`9;O_>l-%G)hRYAjChxV zcMg6+do?x3dw$JRbDaccEH!*!T!bmkN;L=J@2S25b-MnfrbBIfro)zI(pU;?^x5hzJe2hTs;k9XsrnqcEY?AcxrNDDmRt`|LaTQhZ18o0ltMBpe ztE!6DQhuM*I}b9!Ws`5X>J5T=xS99d{~l>NU33=|Vvm5+XAaPh<8jRgofk zaGeL!z)^MCI0*}@w?8Y^X`(X<& zC<4=5Jatb=rwc>+CLQYn)WjsS=<#o!OxbR#ix0aeL)qm}c6b3U6Fj{fNulNU$~A~e zCPRC*x(f3AmJ7Zd;EMp>bK!RiIHp(r1rJAbe|$;kMDCd?BKKnCuc*lvf&4JY2j!ow zN=~t5@-v7^Za{lA^6{Q$x#WjIz6s>r_>K^P=$3SZk!d1F@Mf8JRXM`$O~#7#_!xm$ z$*z-B+p)B?7e;0RF}LPt2y6}Y!P;qC1ZxD($I3`pOk zV_ksi>K8N-1UYE0_8#uz%arY`s;jrRL)llM>^s@-D$3&43JNXPm35}j5iW4SUj_I( z0B_EB1S6u8BM>@~PeB{n+V~8M~DU?IRY^PwbBvV%Q&eEP+ir@71foVDcfFES9`uPR%~biWv$aN zaDO^RpwMz%IUp*H4`{FUP8;O;BM*E-3xNNG+wlJ4J3=g?lOqs1k#7`|_hRHPtI2-? zxi>zD>ZjD{Rmmx~Om4~(I>JRRxi`pfgZvWT5e6W-B^}`w7^yf|fv9h?7~Y#@u2VU} zp4-SAkN%Gy?{H4Z9F9P2lGNr=<`0%|gur~EBV6LDhmQ}~8v^yF@g2bgLUIIR1Zt%t ze2R>M{q6>I>KeH!22QaeeK&X z zK@Nx1di>--M7N|PEH4r{LSL47Ta_d1YhkR|h()qdr^;t2nZpr?4UhDE*?hqgj$lTSfw2kgU5ky;D86F^9gK#X)ygS9QXDH^~L zs0C~_!4YhR2p!=HSKtzT!hSqMNNZvDt4JRYq@^xEb+sH#Xgfk6Q?|9LuJ*Tuva_M= z#MfdKW#I@ETCS`25mn#_zjDDbr??->C$w(Ad=1gb5eS{gjU^)TAV%IwP3{D8tY3-B zd^NBtImMRATa*eN;VPGWC&+t&{2+gfFbL5t=?DkFs69pqW|_BCIl}&4#)?hPA@hdy zTPc~t5r|Ea+AS*nuOBu?$Hcx&=m@{@)Z2uQ57<8o>ZS3=2#FvhM<7O^Ryx8Sw506_ zAxwi7Do5Bq3mUlNBa(i7p`M}v9D!QERudedQMu3&e&-78u1`477t>sO_-75TBYl&O zbpbkt%g}_jBlKg++N9KToh&D2ozebt9^(naD+d&;0AzWaj4e9?j%Gf zM<8?}_ZuoA@6X7atI4B5o(uA=CWorz6k8@gkEnt%!Zj{=F36KXK9BDRgAv`5j_}4X zkt2k%%wJSF!hvLD{t+^d>@!-)9F9P2lJsTE@f$5MMrb}l=m>vu)x#tI5A20{{(MJB z1|c~DF#@&H5%!@aZAS=W8Z=Wm!hyZe;082!rO_ot12_V;fUPDtLfer-N4U-v_=Y~= zU^=F`X7U{&1?ih~tP4MjP7X5sbX4n!F6;ui%4|ZWtd{C8yXj`NdH}N4Uu) ze+A?dL0-Ujgj7Vgq$5nM5II65%e;xo5e`m7=6G_sVOAUp>FLZOu}RXI#VN}y;Rv@u zCYV|Li>n?U`G4>N)N8|cgftM6BM>7{D;;6>XptirmR>N4STy1v87de{ED6>({3s7B686&DI zBU84qs;&-Ag|Z()S!4Jwin16ZP-wZX)+4H5jBuL^{vp7Z0{m6JBV-^tIRc>*x%*fV zc@!gmPEEcP;EEXcVB^@Dmn#d6nSmt$Aj&S5IGRNckBNEc~Dw)F(h)t3XwjTVwCB_Kf zf=u8DkGbmMk^k?6YR@bdfsh=57=c>p2)WZmj_@keppMEB-U)>UxV|0X{UBV?0FFQ{ zV5) zN)=_{2ozebt2Yo;+Y!7Z9{4sZeZtX(+B1to5S<)>(24xDnIiH*jJ%eL{3sS}ZF>nH zcJxS}R+XG$%j8XFi5$Vpic9_y$XjdAES4a;B^_Ze7`4wD5?SWfDn~fl8kvtk=F4tj z#{TI%0v`f4OT*fp?`N)G=L*e3)pIcBh;QFas;nhT!B~W6OOgho>?qI`X(Lg0#sKU(1iBb zJBcYPsp{%jJ1Bbu%D(1{M~XbXLy1Dmb#)w31&&ag3w{LPuG%w;<%mv>Khe_E?f})+L$wAA|4~%yg};YtQhl9n7^!CcZ|n3;tgsR2wREM8 z6p7!Vy_#NrDNL__-cjkb9(oOeUWb3GQ1oiy?oMr^HA?iJ3tOID8l*+OWS#Jh)`qLz5`DsP52!biKRq{s)U&GH zE>~GDRR6zPF7(=Z(Hm_lQ}^N6XVpFK0d*fh-FC50tq?AF2B6a-l^a zLwL4VE3W{r`dsxMKs{XUxZ8E%d8NEsBM4QJjZQZbgeHF?q~6_ggN=lRk8H2jv-+g5 z+#eiO<&KLLJ6=TYb!MGYdX~MrJ4MF(D|Xr_`b0$6K6UW2atNZP)KKa-0!5yV+HcSr;mhQ{Oz8S2L=k%0=AkkP)I`Bg7@17T!HcN zC?|}VzfI!L+mAx}^nUwq7iqdl^nRPcq`jwlzdd1uv~iI3gtJbO7UvEWTkaa4A?h5w zroCF%9`DtVOCAUEB#>*YBdkF5XGh@woiXg*EHZ{nmib+kF`P(3=6jI2`N2^oa~K1$ zNz#0)AIDi)Is6B9`>Zk8Z4nL-?YQdgfqHLiPb-fGp-IOGsdnE+6WRmBESCHo^odsnw_skTlxmJU|v(}N{h|34(sVa5Oe literal 0 HcmV?d00001 diff --git a/testing/btest/scripts/base/protocols/ncp/event.bro b/testing/btest/scripts/base/protocols/ncp/event.bro new file mode 100644 index 0000000000..acb4bf0a0c --- /dev/null +++ b/testing/btest/scripts/base/protocols/ncp/event.bro @@ -0,0 +1,20 @@ +# @TEST-EXEC: bro -C -r $TRACES/ncp.pcap %INPUT >out +# @TEST-EXEC: btest-diff out + +redef likely_server_ports += { 524/tcp }; + +event bro_init() + { + const ports = { 524/tcp }; + Analyzer::register_for_ports(Analyzer::ANALYZER_NCP, ports); + } + +event ncp_request(c: connection, frame_type: count, length: count, func: count) + { + print "ncp request", frame_type, length, func; + } + +event ncp_reply(c: connection, frame_type: count, length: count, req_frame: count, req_func: count, completion_code: count) + { + print "ncp reply", frame_type, length, req_frame, req_func, completion_code; + } From 58864c358ccd1a37c112d15acaa0742d0846336c Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 22 May 2018 18:27:52 -0500 Subject: [PATCH 15/56] Add NCP::max_frame_size tuning option This helps prevent excessive allocations based on message lengths taken from NCP headers. --- scripts/base/init-bare.bro | 6 + src/analyzer/protocol/ncp/CMakeLists.txt | 2 +- src/analyzer/protocol/ncp/NCP.cc | 44 +- src/analyzer/protocol/ncp/NCP.h | 11 +- src/analyzer/protocol/ncp/consts.bif | 1 + .../out | 418 ++++++++++++++++++ .../base/protocols/ncp/frame_size_tuning.bro | 20 + 7 files changed, 487 insertions(+), 15 deletions(-) create mode 100644 src/analyzer/protocol/ncp/consts.bif create mode 100644 testing/btest/Baseline/scripts.base.protocols.ncp.frame_size_tuning/out create mode 100644 testing/btest/scripts/base/protocols/ncp/frame_size_tuning.bro diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index d5bb8f2be9..e592f9277e 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -4806,6 +4806,12 @@ export { const max_frag_data = 30000 &redef; } +module NCP; +export { + ## The maximum number of bytes to allocate when parsing NCP frames. + const max_frame_size = 65536 &redef; +} + module Cluster; export { type Cluster::Pool: record {}; diff --git a/src/analyzer/protocol/ncp/CMakeLists.txt b/src/analyzer/protocol/ncp/CMakeLists.txt index bd06d4e426..1ec5cf2e67 100644 --- a/src/analyzer/protocol/ncp/CMakeLists.txt +++ b/src/analyzer/protocol/ncp/CMakeLists.txt @@ -5,6 +5,6 @@ include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DI bro_plugin_begin(Bro NCP) bro_plugin_cc(NCP.cc Plugin.cc) -bro_plugin_bif(events.bif) +bro_plugin_bif(events.bif consts.bif) bro_plugin_pac(ncp.pac) bro_plugin_end() diff --git a/src/analyzer/protocol/ncp/NCP.cc b/src/analyzer/protocol/ncp/NCP.cc index f2745666dc..f01c409429 100644 --- a/src/analyzer/protocol/ncp/NCP.cc +++ b/src/analyzer/protocol/ncp/NCP.cc @@ -9,6 +9,7 @@ #include "NCP.h" #include "events.bif.h" +#include "consts.bif.h" using namespace std; using namespace analyzer::ncp; @@ -105,13 +106,12 @@ void FrameBuffer::Reset() msg_len = 0; } -// Returns true if we have a complete frame -bool FrameBuffer::Deliver(int &len, const u_char* &data) +int FrameBuffer::Deliver(int &len, const u_char* &data) { ASSERT(buf_len >= hdr_len); if ( len == 0 ) - return false; + return -1; if ( buf_n < hdr_len ) { @@ -123,13 +123,16 @@ bool FrameBuffer::Deliver(int &len, const u_char* &data) } if ( buf_n < hdr_len ) - return false; + return -1; compute_msg_length(); if ( msg_len > buf_len ) { - buf_len = msg_len * 2; + if ( msg_len > BifConst::NCP::max_frame_size ) + return 1; + + buf_len = msg_len; u_char* new_buf = new u_char[buf_len]; memcpy(new_buf, msg_buf, buf_n); delete [] msg_buf; @@ -143,7 +146,13 @@ bool FrameBuffer::Deliver(int &len, const u_char* &data) ++buf_n; ++data; --len; } - return buf_n >= msg_len; + if ( buf_n < msg_len ) + return -1; + + if ( buf_n == msg_len ) + return 0; + + return 1; } void NCP_FrameBuffer::compute_msg_length() @@ -203,10 +212,27 @@ void Contents_NCP_Analyzer::DeliverStream(int len, const u_char* data, bool orig resync = false; } - while ( buffer.Deliver(len, data) ) + for ( ; ; ) { - session->Deliver(IsOrig(), buffer.Len(), buffer.Data()); - buffer.Reset(); + auto result = buffer.Deliver(len, data); + + if ( result < 0 ) + break; + + if ( result == 0 ) + { + session->Deliver(IsOrig(), buffer.Len(), buffer.Data()); + buffer.Reset(); + } + else + { + // The rest of the data available in this delivery will + // be discarded and will need to resync to a new frame header. + Weird("ncp_large_frame"); + buffer.Reset(); + resync = true; + break; + } } } diff --git a/src/analyzer/protocol/ncp/NCP.h b/src/analyzer/protocol/ncp/NCP.h index f8cac95090..bdf5d8bffe 100644 --- a/src/analyzer/protocol/ncp/NCP.h +++ b/src/analyzer/protocol/ncp/NCP.h @@ -54,8 +54,9 @@ public: explicit FrameBuffer(int header_length); virtual ~FrameBuffer(); - // Returns true if a frame is ready - bool Deliver(int& len, const u_char* &data); + // Returns -1 if frame is not ready, 0 if it else, and 1 if + // the frame would require too large of a buffer allocation. + int Deliver(int& len, const u_char* &data); void Reset(); @@ -68,9 +69,9 @@ protected: int hdr_len; u_char* msg_buf; - int msg_len; - int buf_n; // number of bytes in msg_buf - int buf_len; // size off msg_buf + uint64 msg_len; + size_t buf_n; // number of bytes in msg_buf + size_t buf_len; // size off msg_buf }; #define NCP_TCPIP_HEADER_LENGTH 8 diff --git a/src/analyzer/protocol/ncp/consts.bif b/src/analyzer/protocol/ncp/consts.bif new file mode 100644 index 0000000000..452dd9a2b6 --- /dev/null +++ b/src/analyzer/protocol/ncp/consts.bif @@ -0,0 +1 @@ +const NCP::max_frame_size: count; diff --git a/testing/btest/Baseline/scripts.base.protocols.ncp.frame_size_tuning/out b/testing/btest/Baseline/scripts.base.protocols.ncp.frame_size_tuning/out new file mode 100644 index 0000000000..cfb805ee70 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.ncp.frame_size_tuning/out @@ -0,0 +1,418 @@ +ncp reply, 13107, 70, 0, 0, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 86, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 79, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 86, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 79, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 86, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 79, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 86, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 79, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 86, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 79, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 86, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 59, 89 +ncp reply, 13107, 2, 8738, 89, 255 +ncp request, 8738, 59, 89 +ncp reply, 13107, 2, 8738, 89, 255 +ncp request, 8738, 79, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp reply, 13107, 86, 8738, 72, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 66, 89 +ncp reply, 13107, 92, 8738, 89, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 66, 89 +ncp reply, 13107, 92, 8738, 89, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 14, 72 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 46, 89 +ncp reply, 13107, 88, 8738, 89, 0 +ncp request, 8738, 40, 89 +ncp reply, 13107, 11, 8738, 89, 0 +ncp request, 8738, 40, 89 +ncp reply, 13107, 102, 8738, 89, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 6, 22338 +ncp reply, 13107, 10, 8738, 22338, 0 +ncp request, 8738, 14, 72 +ncp request, 8738, 8, 66 +ncp reply, 13107, 2, 8738, 66, 0 +ncp request, 8738, 72, 89 +ncp reply, 13107, 70, 8738, 89, 0 +ncp request, 8738, 7, 22306 +ncp reply, 13107, 2, 8738, 22306, 0 +ncp request, 8738, 14, 72 +ncp request, 8738, 14, 72 diff --git a/testing/btest/scripts/base/protocols/ncp/frame_size_tuning.bro b/testing/btest/scripts/base/protocols/ncp/frame_size_tuning.bro new file mode 100644 index 0000000000..46ad87e752 --- /dev/null +++ b/testing/btest/scripts/base/protocols/ncp/frame_size_tuning.bro @@ -0,0 +1,20 @@ +# @TEST-EXEC: bro -C -r $TRACES/ncp.pcap %INPUT NCP::max_frame_size=150 >out +# @TEST-EXEC: btest-diff out + +redef likely_server_ports += { 524/tcp }; + +event bro_init() + { + const ports = { 524/tcp }; + Analyzer::register_for_ports(Analyzer::ANALYZER_NCP, ports); + } + +event ncp_request(c: connection, frame_type: count, length: count, func: count) + { + print "ncp request", frame_type, length, func; + } + +event ncp_reply(c: connection, frame_type: count, length: count, req_frame: count, req_func: count, completion_code: count) + { + print "ncp reply", frame_type, length, req_frame, req_func, completion_code; + } From 08c64112f016f60d80df7f085fa3874cd8589db7 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 31 May 2018 13:12:46 -0500 Subject: [PATCH 16/56] Document variable argument list BIFs using ellipsis Instead of a single parameter: "va_args: any". --- src/Type.cc | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/Type.cc b/src/Type.cc index aa9388d64e..7ddca8f907 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -1185,7 +1185,14 @@ void RecordType::DescribeFieldsReST(ODesc* d, bool func_args) const if ( d->FindType(td->type) ) d->Add(""); else - td->DescribeReST(d); + { + if ( num_fields == 1 && streq(td->id, "va_args") && + td->type->Tag() == TYPE_ANY ) + // This was a BIF using variable argument list + d->Add("..."); + else + td->DescribeReST(d); + } if ( func_args ) continue; From 224ee790e2e4894c2083371185b60593169f5be8 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 31 May 2018 15:26:22 -0500 Subject: [PATCH 17/56] Add Broker::publish_and_relay BIF Like Broker::relay, except the relaying-node also calls event handlers. --- aux/broker | 2 +- src/broker/Manager.cc | 65 +++++++-- src/broker/Manager.h | 22 ++- src/broker/messaging.bif | 72 +++++++++- .../one.one.out | 3 + .../three.three.out | 2 + .../two.two.out | 5 + .../broker/remote_publish_and_relay_event.bro | 125 ++++++++++++++++++ 8 files changed, 270 insertions(+), 26 deletions(-) create mode 100644 testing/btest/Baseline/broker.remote_publish_and_relay_event/one.one.out create mode 100644 testing/btest/Baseline/broker.remote_publish_and_relay_event/three.three.out create mode 100644 testing/btest/Baseline/broker.remote_publish_and_relay_event/two.two.out create mode 100644 testing/btest/broker/remote_publish_and_relay_event.bro diff --git a/aux/broker b/aux/broker index 7b84848bde..11c9313582 160000 --- a/aux/broker +++ b/aux/broker @@ -1 +1 @@ -Subproject commit 7b84848bded443637fa34e76f7d8558bd1cafbee +Subproject commit 11c9313582b5505c002a62eba28ffb93bae1e5b3 diff --git a/src/broker/Manager.cc b/src/broker/Manager.cc index 47f7c95722..8184a86111 100644 --- a/src/broker/Manager.cc +++ b/src/broker/Manager.cc @@ -341,7 +341,8 @@ bool Manager::PublishEvent(string topic, RecordVal* args) bool Manager::RelayEvent(std::string first_topic, broker::set relay_topics, std::string name, - broker::vector args) + broker::vector args, + bool handle_on_relayer) { if ( bstate->endpoint.is_shutdown() ) return true; @@ -349,18 +350,33 @@ bool Manager::RelayEvent(std::string first_topic, if ( ! bstate->endpoint.peers().size() ) return true; - DBG_LOG(DBG_BROKER, "Publishing relay event: %s", + DBG_LOG(DBG_BROKER, "Publishing %s-relay event: %s", + handle_on_relayer ? "handle" : "", RenderEvent(first_topic, name, args).c_str()); - broker::bro::RelayEvent msg(std::move(relay_topics), std::move(name), - std::move(args)); - bstate->endpoint.publish(std::move(first_topic), std::move(msg)); + + if ( handle_on_relayer ) + { + broker::bro::HandleAndRelayEvent msg(std::move(relay_topics), + std::move(name), + std::move(args)); + bstate->endpoint.publish(std::move(first_topic), std::move(msg)); + } + else + { + broker::bro::RelayEvent msg(std::move(relay_topics), + std::move(name), + std::move(args)); + bstate->endpoint.publish(std::move(first_topic), std::move(msg)); + } + ++statistics.num_events_outgoing; return true; } bool Manager::RelayEvent(std::string first_topic, std::set relay_topics, - RecordVal* args) + RecordVal* args, + bool handle_on_relayer) { if ( bstate->endpoint.is_shutdown() ) return true; @@ -389,7 +405,7 @@ bool Manager::RelayEvent(std::string first_topic, topic_set.emplace(std::move(t)); return RelayEvent(first_topic, std::move(topic_set), event_name, - std::move(xs)); + std::move(xs), handle_on_relayer); } bool Manager::PublishIdentifier(std::string topic, std::string id) @@ -820,6 +836,10 @@ void Manager::DispatchMessage(broker::data msg) ProcessRelayEvent(std::move(msg)); break; + case broker::bro::Message::Type::HandleAndRelayEvent: + ProcessHandleAndRelayEvent(std::move(msg)); + break; + case broker::bro::Message::Type::LogCreate: ProcessLogCreate(std::move(msg)); break; @@ -907,23 +927,23 @@ void Manager::Process() SetIdle(! had_input); } -void Manager::ProcessEvent(broker::bro::Event ev) + +void Manager::ProcessEvent(std::string name, broker::vector args) { - DBG_LOG(DBG_BROKER, "Received event: %s", RenderMessage(ev).c_str()); - + DBG_LOG(DBG_BROKER, "Process event: %s %s", + name.data(), RenderMessage(args).data()); ++statistics.num_events_incoming; + auto handler = event_registry->Lookup(name.data()); - auto handler = event_registry->Lookup(ev.name().c_str()); if ( ! handler ) return; - auto& args = ev.args(); auto arg_types = handler->FType(false)->ArgTypes()->Types(); if ( static_cast(arg_types->length()) != args.size() ) { reporter->Warning("got event message '%s' with invalid # of args," - " got %zd, expected %d", ev.name().data(), args.size(), + " got %zd, expected %d", name.data(), args.size(), arg_types->length()); return; } @@ -942,7 +962,7 @@ void Manager::ProcessEvent(broker::bro::Event ev) { reporter->Warning("failed to convert remote event '%s' arg #%d," " got %s, expected %s", - ev.name().data(), i, got_type, + name.data(), i, got_type, type_name(expected_type->Tag())); break; } @@ -954,6 +974,11 @@ void Manager::ProcessEvent(broker::bro::Event ev) delete_vals(vl); } +void Manager::ProcessEvent(broker::bro::Event ev) + { + ProcessEvent(std::move(ev.name()), std::move(ev.args())); + } + void Manager::ProcessRelayEvent(broker::bro::RelayEvent ev) { DBG_LOG(DBG_BROKER, "Received relay event: %s", RenderMessage(ev).c_str()); @@ -965,6 +990,18 @@ void Manager::ProcessRelayEvent(broker::bro::RelayEvent ev) std::move(ev.args())); } +void Manager::ProcessHandleAndRelayEvent(broker::bro::HandleAndRelayEvent ev) + { + DBG_LOG(DBG_BROKER, "Received handle-relay event: %s", + RenderMessage(ev).c_str()); + ProcessEvent(ev.name(), ev.args()); + + for ( auto& t : ev.topics() ) + PublishEvent(std::move(broker::get(t)), + std::move(ev.name()), + std::move(ev.args())); + } + bool bro_broker::Manager::ProcessLogCreate(broker::bro::LogCreate lc) { DBG_LOG(DBG_BROKER, "Received log-create: %s", RenderMessage(lc).c_str()); diff --git a/src/broker/Manager.h b/src/broker/Manager.h index 996859636d..976aa82a03 100644 --- a/src/broker/Manager.h +++ b/src/broker/Manager.h @@ -148,33 +148,41 @@ public: bool PublishEvent(std::string topic, RecordVal* ev); /** - * Sends an event to any interested peers, who, upon receipt, immediately - * republish the event to a new set of topics. + * Sends an event to any interested peers, who, upon receipt, + * republishes the event to a new set of topics and optionally + * calls event handlers. * @param first_topic the first topic to use when publishing the event * @param relay_topics the set of topics the receivers will use to * republish the event. The event is relayed at most a single hop. * @param name the name of the event * @param args the event's arguments + * @param handle_on_relayer whether they relaying-node should call event + * handlers. * @return true if the message is sent successfully. */ bool RelayEvent(std::string first_topic, broker::set relay_topics, std::string name, - broker::vector args); + broker::vector args, + bool handle_on_relayer); /** - * Sends an event to any interested peers, who, upon receipt, immediately - * republish the event to a new set of topics. + * Sends an event to any interested peers, who, upon receipt, + * republishes the event to a new set of topics and optionally + * calls event handlers. * @param first_topic the first topic to use when publishing the event * @param relay_topics the set of topics the receivers will use to * republish the event. The event is relayed at most a single hop. * @param ev the event and its arguments to send to peers, in the form of * a Broker::Event record type. + * @param handle_on_relayer whether they relaying-node should call event + * handlers. * @return true if the message is sent successfully. */ bool RelayEvent(std::string first_topic, std::set relay_topics, - RecordVal* ev); + RecordVal* ev, + bool handle_on_relayer); /** * Send a message to create a log stream to any interested peers. @@ -340,8 +348,10 @@ private: }; void DispatchMessage(broker::data msg); + void ProcessEvent(std::string name, broker::vector args); void ProcessEvent(broker::bro::Event ev); void ProcessRelayEvent(broker::bro::RelayEvent re); + void ProcessHandleAndRelayEvent(broker::bro::HandleAndRelayEvent ev); bool ProcessLogCreate(broker::bro::LogCreate lc); bool ProcessLogWrite(broker::bro::LogWrite lw); bool ProcessIdentifierUpdate(broker::bro::IdentifierUpdate iu); diff --git a/src/broker/messaging.bif b/src/broker/messaging.bif index 8b2c64e86f..c7b16dba72 100644 --- a/src/broker/messaging.bif +++ b/src/broker/messaging.bif @@ -76,13 +76,13 @@ static bool relay_event_args(val_list& args, const BroString* topic, if ( args[0]->Type()->Tag() == TYPE_RECORD ) rval = broker_mgr->RelayEvent(topic->CheckString(), std::move(topic_set), - args[0]->AsRecordVal()); + args[0]->AsRecordVal(), false); else { auto ev = broker_mgr->MakeEvent(&args, frame); rval = broker_mgr->RelayEvent(topic->CheckString(), std::move(topic_set), - ev); + ev, false); Unref(ev); } @@ -133,7 +133,7 @@ function Broker::publish%(topic: string, ...%): bool ## Publishes an event at a given topic, with any receivers automatically ## forwarding it to its peers with a different topic. The event is relayed -## at most a single hop. +## at most a single hop and the relayer does not call any local event handlers. ## ## first_topic: the initial topic to use for publishing the event. ## @@ -181,12 +181,74 @@ function Broker::relay%(first_topic: string, ...%): bool if ( args[0]->Type()->Tag() == TYPE_RECORD ) rval = broker_mgr->RelayEvent(first_topic->CheckString(), std::move(topic_set), - args[0]->AsRecordVal()); + args[0]->AsRecordVal(), false); else { auto ev = broker_mgr->MakeEvent(&args, frame); rval = broker_mgr->RelayEvent(first_topic->CheckString(), - std::move(topic_set), ev); + std::move(topic_set), ev, false); + Unref(ev); + } + + return new Val(rval, TYPE_BOOL); + %} + +## Publishes an event at a given topic, with any receivers automatically +## forwarding it to its peers with a different topic. The event is relayed +## at most a single hop and the relayer does call local event handlers. +## +## first_topic: the initial topic to use for publishing the event. +## +## args: the first member of the argument list may be either a string or +## a set of strings indicating the secondary topic that the first +## set of receivers will use to re-publish the event. The remaining +## members of the argument list are either the return value of a +## previously-made call to :bro:see:`Broker::make_event` or the +## argument list that should be passed along to it, so that it can +## be called as part of executing this function. +## +## Returns: true if the message is sent. +function Broker::publish_and_relay%(first_topic: string, ...%): bool + %{ + bro_broker::Manager::ScriptScopeGuard ssg; + val_list* bif_args = @ARGS@; + + if ( bif_args->length() < 3 ) + { + builtin_error("Broker::publish_and_relay requires at least 3 arguments"); + return new Val(false, TYPE_BOOL); + } + + auto second_topic = (*bif_args)[1]; + + if ( second_topic->Type()->Tag() != TYPE_STRING && + ! is_string_set(second_topic->Type()) ) + { + builtin_error("Broker::publish_and_relay requires a string or string_set as 2nd argument"); + return new Val(false, TYPE_BOOL); + } + + auto topic_set = val_to_topic_set(second_topic); + + if ( topic_set.empty() ) + return new Val(false, TYPE_BOOL); + + val_list args(bif_args->length() - 2); + + for ( auto i = 2; i < bif_args->length(); ++i ) + args.append((*bif_args)[i]); + + auto rval = false; + + if ( args[0]->Type()->Tag() == TYPE_RECORD ) + rval = broker_mgr->RelayEvent(first_topic->CheckString(), + std::move(topic_set), + args[0]->AsRecordVal(), true); + else + { + auto ev = broker_mgr->MakeEvent(&args, frame); + rval = broker_mgr->RelayEvent(first_topic->CheckString(), + std::move(topic_set), ev, true); Unref(ev); } diff --git a/testing/btest/Baseline/broker.remote_publish_and_relay_event/one.one.out b/testing/btest/Baseline/broker.remote_publish_and_relay_event/one.one.out new file mode 100644 index 0000000000..45c18d28be --- /dev/null +++ b/testing/btest/Baseline/broker.remote_publish_and_relay_event/one.one.out @@ -0,0 +1,3 @@ +sender added peer: endpoint=127.0.0.1 msg=received handshake from remote core +got ready event +sender lost peer: endpoint=127.0.0.1 msg=lost remote peer diff --git a/testing/btest/Baseline/broker.remote_publish_and_relay_event/three.three.out b/testing/btest/Baseline/broker.remote_publish_and_relay_event/three.three.out new file mode 100644 index 0000000000..8193829fd4 --- /dev/null +++ b/testing/btest/Baseline/broker.remote_publish_and_relay_event/three.three.out @@ -0,0 +1,2 @@ +receiver added peer: endpoint=127.0.0.1 msg=handshake successful +got my_event, hello world diff --git a/testing/btest/Baseline/broker.remote_publish_and_relay_event/two.two.out b/testing/btest/Baseline/broker.remote_publish_and_relay_event/two.two.out new file mode 100644 index 0000000000..7bedece7d2 --- /dev/null +++ b/testing/btest/Baseline/broker.remote_publish_and_relay_event/two.two.out @@ -0,0 +1,5 @@ +receiver added peer: endpoint=127.0.0.1 msg=received handshake from remote core +receiver added peer: endpoint=127.0.0.1 msg=handshake successful +sending ready event +got my_event, hello world +receiver lost peer: endpoint=127.0.0.1 msg=lost remote peer diff --git a/testing/btest/broker/remote_publish_and_relay_event.bro b/testing/btest/broker/remote_publish_and_relay_event.bro new file mode 100644 index 0000000000..493e673af2 --- /dev/null +++ b/testing/btest/broker/remote_publish_and_relay_event.bro @@ -0,0 +1,125 @@ +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run three "bro -B broker -b ../three.bro >three.out" +# @TEST-EXEC: btest-bg-run two "bro -B broker -b ../two.bro >two.out" +# @TEST-EXEC: btest-bg-run one "bro -B broker -b ../one.bro >one.out" +# +# @TEST-EXEC: btest-bg-wait 20 +# @TEST-EXEC: btest-diff one/one.out +# @TEST-EXEC: btest-diff two/two.out +# @TEST-EXEC: btest-diff three/three.out + +@TEST-START-FILE one.bro + +redef Broker::default_connect_retry=1secs; +redef Broker::default_listen_retry=1secs; +redef exit_only_after_terminate = T; + +event my_event(s: string) + { + print "got my_event", s; + } + +event ready_event() + { + print "got ready event"; + + Broker::publish_and_relay("bro/event/pre-relay", "bro/event/post-relay", + my_event, "hello world"); + } + +event bro_init() + { + Broker::subscribe("bro/event/ready"); + Broker::peer("127.0.0.1", 10000/tcp); + } + +event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string) + { + print fmt("sender added peer: endpoint=%s msg=%s", + endpoint$network$address, msg); + } + +event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string) + { + print fmt("sender lost peer: endpoint=%s msg=%s", + endpoint$network$address, msg); + terminate(); + } + +@TEST-END-FILE + + +@TEST-START-FILE two.bro + +redef Broker::default_connect_retry=1secs; +redef Broker::default_listen_retry=1secs; +redef exit_only_after_terminate = T; + +global peers_added = 0; + +event my_event(s: string) + { + print "got my_event", s; + } + +event ready_event() + { + } + +event bro_init() + { + Broker::subscribe("bro/event/pre-relay"); + Broker::listen("127.0.0.1", 10000/tcp); + Broker::peer("127.0.0.1", 9999/tcp); + } + +event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string) + { + print fmt("receiver added peer: endpoint=%s msg=%s", endpoint$network$address, msg); + ++peers_added; + + if ( peers_added == 2 ) + { + print "sending ready event"; + Broker::publish("bro/event/ready", ready_event); + } + } + +event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string) + { + print fmt("receiver lost peer: endpoint=%s msg=%s", endpoint$network$address, msg); + terminate(); + } + +@TEST-END-FILE + +@TEST-START-FILE three.bro + +redef Broker::default_connect_retry=1secs; +redef Broker::default_listen_retry=1secs; +redef exit_only_after_terminate = T; + +event my_event(s: string) + { + print "got my_event", s; + terminate(); + } + +event bro_init() + { + Broker::subscribe("bro/event/post-relay"); + Broker::listen("127.0.0.1", 9999/tcp); + } + +event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string) + { + print fmt("receiver added peer: endpoint=%s msg=%s", endpoint$network$address, msg); + } + +event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string) + { + print fmt("receiver lost peer: endpoint=%s msg=%s", endpoint$network$address, msg); + } + +@TEST-END-FILE From 1fec186c39cbfe814b8d6e2cbbfff3e9397d17d7 Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Tue, 22 May 2018 17:18:14 -0700 Subject: [PATCH 18/56] Fix SCT validation when invalid certificates are in chain. At the moment it would try to access an unset optional in this case. --- scripts/policy/protocols/ssl/validate-sct.bro | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/policy/protocols/ssl/validate-sct.bro b/scripts/policy/protocols/ssl/validate-sct.bro index a89a5e5b19..f4d1646ae8 100644 --- a/scripts/policy/protocols/ssl/validate-sct.bro +++ b/scripts/policy/protocols/ssl/validate-sct.bro @@ -180,6 +180,8 @@ hook ssl_finishing(c: connection) &priority=19 { if ( i == 0 ) # end-host-cert next; + if ( ! c$ssl$cert_chain[i]?$x509 || ! c$ssl$cert_chain[i]$x509?$handle ) + next; issuer_key_hash = x509_spki_hash(c$ssl$cert_chain[i]$x509$handle, 4); valid = sct_verify(cert, proof$logid, log$key, proof$signature, proof$timestamp, proof$hash_alg, issuer_key_hash); From f9e5777e6fc2a3f9ea19b042b73574949345bad8 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 1 Jun 2018 10:03:24 -0500 Subject: [PATCH 19/56] BIT-1635: fix `make doc` warnings References to Input::Reader and Log::Writer enum types no longer emit warnings as they are now hardcoded to be documented as part of their associated framework scripts and so links to them now work. --- CHANGES | 10 ++++++++++ VERSION | 2 +- doc/conf.py.in | 2 -- src/broxygen/ScriptInfo.cc | 13 +++++++++++++ 4 files changed, 24 insertions(+), 3 deletions(-) diff --git a/CHANGES b/CHANGES index 6b81d44a0e..6d57a8caa8 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,14 @@ +2.5-618 | 2018-06-01 10:03:24 -0500 + + * BIT-1635: fix `make doc` warnings (Corelight) + + * Add smb2_file_sattr event (Devin Trejo) + + * Add bad ARP tests (Pierre LATET) + + * Fix SCT validation when invalid certificates are in chain. (Johanna Amann) + 2.5-611 | 2018-05-29 10:13:17 -0500 * Fix NEWS file formatting (Corelight) diff --git a/VERSION b/VERSION index 67d2dc9fde..05bbcf298c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.5-611 +2.5-618 diff --git a/doc/conf.py.in b/doc/conf.py.in index ef9367483a..f7243b4527 100644 --- a/doc/conf.py.in +++ b/doc/conf.py.in @@ -195,8 +195,6 @@ html_sidebars = { # Output file base name for HTML help builder. htmlhelp_basename = 'Broxygen' -html_add_permalinks = None - # -- Options for LaTeX output -------------------------------------------------- # The paper size ('letter' or 'a4'). diff --git a/src/broxygen/ScriptInfo.cc b/src/broxygen/ScriptInfo.cc index 0a57991969..2c054ea9b1 100644 --- a/src/broxygen/ScriptInfo.cc +++ b/src/broxygen/ScriptInfo.cc @@ -250,6 +250,19 @@ void ScriptInfo::DoInitPostScript() id->Name(), name.c_str()); state_vars.push_back(info); } + + // The following enum types are automatically created internally in Bro, + // so just manually associating them with scripts for now. + if ( name == "base/frameworks/input/main.bro" ) + { + auto id = global_scope()->Lookup("Input::Reader"); + types.push_back(new IdentifierInfo(id, this)); + } + else if ( name == "base/frameworks/logging/main.bro" ) + { + auto id = global_scope()->Lookup("Log::Writer"); + types.push_back(new IdentifierInfo(id, this)); + } } vector ScriptInfo::GetComments() const From 55f14c2eb848bc69048e319223152ba12437bbe5 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 1 Jun 2018 11:29:15 -0500 Subject: [PATCH 20/56] Relocate temporary script coverage files So they don't clutter the top-level of unit test .tmp/ dir. --- CHANGES | 4 ++++ VERSION | 2 +- src/Brofiler.cc | 14 +++++++++++--- testing/btest/Makefile | 5 +++-- testing/btest/btest.cfg | 2 +- testing/external/subdir-btest.cfg | 2 +- 6 files changed, 21 insertions(+), 8 deletions(-) diff --git a/CHANGES b/CHANGES index 6d57a8caa8..ae4972788f 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.5-619 | 2018-06-01 11:29:15 -0500 + + * Relocate temporary script coverage files (Corelight) + 2.5-618 | 2018-06-01 10:03:24 -0500 * BIT-1635: fix `make doc` warnings (Corelight) diff --git a/VERSION b/VERSION index 05bbcf298c..344be7760a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.5-618 +2.5-619 diff --git a/src/Brofiler.cc b/src/Brofiler.cc index e7d8c8fdeb..a31ec469f0 100644 --- a/src/Brofiler.cc +++ b/src/Brofiler.cc @@ -50,10 +50,18 @@ bool Brofiler::WriteStats() char* bf = getenv("BRO_PROFILER_FILE"); if ( ! bf ) return false; - FILE* f; - const char* p = strstr(bf, ".XXXXXX"); + SafeDirname dirname{bf}; - if ( p && ! p[7] ) + if ( ! ensure_intermediate_dirs(dirname.result.data()) ) + { + reporter->Error("Failed to open BRO_PROFILER_FILE destination '%s' for writing", bf); + return false; + } + + FILE* f; + const char* p = strstr(bf, "XXXXXX"); + + if ( p && ! p[6] ) { mode_t old_umask = umask(S_IXUSR | S_IRWXO | S_IRWXG); int fd = mkstemp(bf); diff --git a/testing/btest/Makefile b/testing/btest/Makefile index 56bf8f0a7e..c9bcfff5ee 100644 --- a/testing/btest/Makefile +++ b/testing/btest/Makefile @@ -1,6 +1,7 @@ DIAG=diag.log BTEST=../../aux/btest/btest +SCRIPT_COV=.tmp/script-coverage all: cleanup btest-verbose coverage @@ -15,11 +16,11 @@ btest-brief: @$(BTEST) -j -b -f $(DIAG) coverage: - @../scripts/coverage-calc ".tmp/script-coverage*" coverage.log `pwd`/../../scripts + @../scripts/coverage-calc "$(SCRIPT_COV)/*" coverage.log `pwd`/../../scripts cleanup: @rm -f $(DIAG) - @rm -f .tmp/script-coverage* + @rm -rf $(SCRIPT_COV)* distclean: cleanup @rm -rf .btest.failed.dat \ diff --git a/testing/btest/btest.cfg b/testing/btest/btest.cfg index 2b9d75287f..8a69f5eddc 100644 --- a/testing/btest/btest.cfg +++ b/testing/btest/btest.cfg @@ -22,6 +22,6 @@ DIST=%(testbase)s/../.. BUILD=%(testbase)s/../../build TEST_DIFF_CANONIFIER=%(testbase)s/../scripts/diff-canonifier TMPDIR=%(testbase)s/.tmp -BRO_PROFILER_FILE=%(testbase)s/.tmp/script-coverage.XXXXXX +BRO_PROFILER_FILE=%(testbase)s/.tmp/script-coverage/XXXXXX BTEST_RST_FILTER=$SCRIPTS/rst-filter BRO_DNS_FAKE=1 diff --git a/testing/external/subdir-btest.cfg b/testing/external/subdir-btest.cfg index 4315ade850..39aaead17a 100644 --- a/testing/external/subdir-btest.cfg +++ b/testing/external/subdir-btest.cfg @@ -18,7 +18,7 @@ SCRIPTS=%(testbase)s/../scripts SCRIPTS_LOCAL=%(testbase)s/scripts DIST=%(testbase)s/../../.. BUILD=%(testbase)s/../../../build -BRO_PROFILER_FILE=%(testbase)s/.tmp/script-coverage.XXXXXX +BRO_PROFILER_FILE=%(testbase)s/.tmp/script-coverage/XXXXXX BRO_DNS_FAKE=1 # For fedora 21 - they disable MD5 for certificate verification and need setting an environment variable to permit it. OPENSSL_ENABLE_MD5_VERIFY=1 From 8bb76cd3c1d1ae5cfb87c59a71c30cebf9506e55 Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Fri, 1 Jun 2018 12:32:34 -0700 Subject: [PATCH 21/56] KRB: do not set authentication info to nouser. Simply do not set it at all in this case - it is an optional after all... --- src/analyzer/protocol/krb/KRB.cc | 16 +++++++--------- src/analyzer/protocol/krb/krb-analyzer.pac | 3 ++- .../scripts.base.protocols.krb.kinit/output | 2 +- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/analyzer/protocol/krb/KRB.cc b/src/analyzer/protocol/krb/KRB.cc index c3dfaf45b7..4a13aec425 100644 --- a/src/analyzer/protocol/krb/KRB.cc +++ b/src/analyzer/protocol/krb/KRB.cc @@ -73,31 +73,30 @@ void KRB_Analyzer::DeliverPacket(int len, const u_char* data, bool orig, StringVal* KRB_Analyzer::GetAuthenticationInfo(const BroString* principal, const BroString* ciphertext, const bro_uint_t enctype) { - StringVal* ret = new StringVal("nouser"); #ifdef USE_KRB5 if ( !krb_available ) - return ret; + return nullptr; BroString delim("/"); int pos = principal->FindSubstring(&delim); if ( pos == -1 ) { reporter->Warning("KRB: Couldn't parse principal (%s)", principal->CheckString()); - return ret; + return nullptr; } std::unique_ptr service = unique_ptr(principal->GetSubstring(0, pos)); std::unique_ptr hostname = unique_ptr(principal->GetSubstring(pos + 1, -1)); if ( !service || !hostname ) { reporter->Warning("KRB: Couldn't parse principal (%s)", principal->CheckString()); - return ret; + return nullptr; } krb5_principal sprinc; krb5_error_code retval = krb5_sname_to_principal(krb_context, hostname->CheckString(), service->CheckString(), KRB5_NT_SRV_HST, &sprinc); if ( retval ) { reporter->Warning("KRB: Couldn't generate principal name (%s)", krb5_get_error_message(krb_context, retval)); - return ret; + return nullptr; } krb5_ticket tkt; @@ -110,7 +109,7 @@ StringVal* KRB_Analyzer::GetAuthenticationInfo(const BroString* principal, const if ( retval ) { reporter->Warning("KRB: Couldn't decrypt ticket (%s)", krb5_get_error_message(krb_context, retval)); - return ret; + return nullptr; } char* cp; @@ -118,10 +117,9 @@ StringVal* KRB_Analyzer::GetAuthenticationInfo(const BroString* principal, const if ( retval ) { reporter->Warning("KRB: Couldn't unparse name (%s)", krb5_get_error_message(krb_context, retval)); - return ret; + return nullptr; } - free(ret); - ret = new StringVal(cp); + StringVal* ret = new StringVal(cp); krb5_free_unparsed_name(krb_context, cp); #endif diff --git a/src/analyzer/protocol/krb/krb-analyzer.pac b/src/analyzer/protocol/krb/krb-analyzer.pac index 2af70fff98..7c59a6a99e 100644 --- a/src/analyzer/protocol/krb/krb-analyzer.pac +++ b/src/analyzer/protocol/krb/krb-analyzer.pac @@ -247,7 +247,8 @@ refine connection KRB_Conn += { RecordVal* rvticket = proc_ticket(${msg.ticket}); StringVal* authenticationinfo = bro_analyzer()->GetAuthenticationInfo(rvticket->Lookup(2)->AsString(), rvticket->Lookup(4)->AsString(), rvticket->Lookup(3)->AsCount()); - rvticket->Assign(5, authenticationinfo); + if ( authenticationinfo ) + rvticket->Assign(5, authenticationinfo); BifEvent::generate_krb_ap_request(bro_analyzer(), bro_analyzer()->Conn(), rvticket, rv); } diff --git a/testing/btest/Baseline/scripts.base.protocols.krb.kinit/output b/testing/btest/Baseline/scripts.base.protocols.krb.kinit/output index 50876c8d47..20b0b568c3 100644 --- a/testing/btest/Baseline/scripts.base.protocols.krb.kinit/output +++ b/testing/btest/Baseline/scripts.base.protocols.krb.kinit/output @@ -1,3 +1,3 @@ KRB_AP_REQUEST -[pvno=5, realm=VLADG.NET, service_name=krbtgt/VLADG.NET, cipher=18, ciphertext={\x9fY\xd0f\x8dS\xf4I\x88\x04\xfa\xc1\xd8m\xa2\xb7+\xbb\x19\xcag\x0c\x13\xd1g*\xfc\x18\xd1\xb1\x80!\xbd\x85\xec\xf9\x9b\xfa-\x18\xb6\xf5h\x91\xe7\x99\xf4\xdb\x93\xa0\xc7\x90\x1e\xa9\x95v\xd3\x12\xfa,9\x1d\x0b\xd0\xa1\xd25\x0f\x1f[G\xdf\xd0\xbbd\x06$2\xd1\xae\x130qZiY\x07@\xe9\xf9\xff\xa4\x9a\xd4\x09\xf0\x0d\xc1R\x10M\xbdKOV\xfd\xf6\x13\xf6\x9a\x95N\xdf!\xf6x\x94\xd8j\xa5\xdcp\xa8\x04\x99\x02x\xdb$\xd8\xfa_o\x8dV\xc8\x0a\xfe\x00\xf3&c\x0c8\xd1\xd0\xe9\x8e\xab\xfe&\xfe\x00\x8d$\x98I\xe5\x8d\x94rM4%\xd8\xfe\xa9\x08\x06\xc6\x95H7\xf7HCq\xb9\x0d$\x95?\x83B\x82\xdd\xea\xc3f3\xcc\xbb\x09\x0d-\x09;\xa6i%\xcd\xba\x11\xd4\xe0\x12w\xd0G&\xdaj\x82\x7f;\xf3\x1d\x10\xa4l\x06\x16l\x1bc\xa1\xd1\x15!\x00\x8a\xff\x8a\x06\xe7U^:] [use_session_key=F, mutual_required=F] From 327acf6555148aba759bd034d57a07773f17d03f Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Fri, 1 Jun 2018 12:46:26 -0700 Subject: [PATCH 22/56] KRB: do not set keytab by default. Only enable decryption if a user purposefully sets a keytab. --- scripts/base/init-bare.bro | 3 ++- src/analyzer/protocol/krb/KRB.cc | 3 +++ .../.stderr | 0 .../.stdout | 1 + .../base/protocols/krb/smb2_krb_nokeytab.test | 20 +++++++++++++++++++ 5 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 testing/btest/Baseline/scripts.base.protocols.krb.smb2_krb_nokeytab/.stderr create mode 100644 testing/btest/Baseline/scripts.base.protocols.krb.smb2_krb_nokeytab/.stdout create mode 100644 testing/btest/scripts/base/protocols/krb/smb2_krb_nokeytab.test diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index 07ed841f6a..1d64c7c0a3 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -4245,7 +4245,8 @@ export { module KRB; export { - const keytab = "/etc/krb5.keytab" &redef; + ## Kerberos keytab file name. Used to decrypt tickets encountered on the wire. + const keytab = "" &redef; ## KDC Options. See :rfc:`4120` type KRB::KDC_Options: record { ## The ticket to be issued should have its forwardable flag set. diff --git a/src/analyzer/protocol/krb/KRB.cc b/src/analyzer/protocol/krb/KRB.cc index 4a13aec425..5a41b0b99b 100644 --- a/src/analyzer/protocol/krb/KRB.cc +++ b/src/analyzer/protocol/krb/KRB.cc @@ -13,6 +13,9 @@ KRB_Analyzer::KRB_Analyzer(Connection* conn) interp = new binpac::KRB::KRB_Conn(this); #ifdef USE_KRB5 + if ( BifConst::KRB::keytab->Len() == 0 ) + return; // no keytab set + const char* keytab_filename = BifConst::KRB::keytab->CheckString(); if ( access(keytab_filename, R_OK) != 0 ) { diff --git a/testing/btest/Baseline/scripts.base.protocols.krb.smb2_krb_nokeytab/.stderr b/testing/btest/Baseline/scripts.base.protocols.krb.smb2_krb_nokeytab/.stderr new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testing/btest/Baseline/scripts.base.protocols.krb.smb2_krb_nokeytab/.stdout b/testing/btest/Baseline/scripts.base.protocols.krb.smb2_krb_nokeytab/.stdout new file mode 100644 index 0000000000..cf84443e49 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.krb.smb2_krb_nokeytab/.stdout @@ -0,0 +1 @@ +F diff --git a/testing/btest/scripts/base/protocols/krb/smb2_krb_nokeytab.test b/testing/btest/scripts/base/protocols/krb/smb2_krb_nokeytab.test new file mode 100644 index 0000000000..0d2c68d142 --- /dev/null +++ b/testing/btest/scripts/base/protocols/krb/smb2_krb_nokeytab.test @@ -0,0 +1,20 @@ +# This test verifies that without a keytab file no entries are +# created and no errors happen. +# +# @TEST-REQUIRES: grep -q "#define USE_KRB5" $BUILD/bro-config.h +# +# @TEST-COPY-FILE: ${TRACES}/krb/smb2_krb.keytab +# @TEST-EXEC: bro -C -r $TRACES/krb/smb2_krb.pcap %INPUT +# @TEST-EXEC: btest-diff .stdout +# @TEST-EXEC: btest-diff .stderr + +global monitor_ports: set[port] = { 445/tcp, 139/tcp } &redef; + +event bro_init() &priority=5{ + Analyzer::register_for_ports(Analyzer::ANALYZER_SMB, monitor_ports); +} + +event krb_ap_request(c: connection, ticket: KRB::Ticket, opts: KRB::AP_Options){ + print ticket?$authenticationinfo; +} + From 6f3ccd507b2dcd095baf4dbcd004b06088f77ad7 Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Fri, 1 Jun 2018 12:48:38 -0700 Subject: [PATCH 23/56] Allow setting KRB5 root dir in configure --- configure | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/configure b/configure index cf1c901449..ed4cbf3d6e 100755 --- a/configure +++ b/configure @@ -72,6 +72,7 @@ Usage: $0 [OPTION]... [VAR=VALUE]... Optional Packages in Non-Standard Locations: --with-geoip=PATH path to the libGeoIP install root + --with-krb5=PATH path to krb5 install root --with-perftools=PATH path to Google Perftools install root --with-jemalloc=PATH path to jemalloc install root --with-python-lib=PATH path to libpython @@ -250,6 +251,9 @@ while [ $# -ne 0 ]; do --with-geoip=*) append_cache_entry LibGeoIP_ROOT_DIR PATH $optarg ;; + --with-krb5=*) + append_cache_entry LibKrb5_ROOT_DIR PATH $optarg + ;; --with-perftools=*) append_cache_entry GooglePerftools_ROOT_DIR PATH $optarg ;; From 1b4e0116f4aa4f846b4ea85b01a26f066f8e401d Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 1 Jun 2018 15:38:11 -0500 Subject: [PATCH 24/56] Allow BRO_DEFAULT_LISTEN_ADDRESS to control broker listen address This environment variable is now set to listen only on IPv4 loopback when running unit tests (instead of using the default INADDR_ANY). This also moves some of the @loads out from init-bare.bro into a new init-frameworks-and-bifs.bro in order to better support calling BIFs (like `getenv`) from variable initializations in those particular frameworks. --- scripts/base/frameworks/broker/main.bro | 2 +- scripts/base/init-bare.bro | 20 +++----------- scripts/base/init-frameworks-and-bifs.bro | 15 +++++++++++ src/input.h | 1 + src/main.cc | 4 ++- src/scan.l | 25 ++++++++++++++---- .../canonified_loaded_scripts.log | 11 ++++---- .../canonified_loaded_scripts.log | 11 ++++---- testing/btest/Baseline/plugins.hooks/output | 26 ++++++++++++++----- testing/btest/btest.cfg | 1 + 10 files changed, 76 insertions(+), 40 deletions(-) create mode 100644 scripts/base/init-frameworks-and-bifs.bro diff --git a/scripts/base/frameworks/broker/main.bro b/scripts/base/frameworks/broker/main.bro index 451d4cf86b..5368d5422b 100644 --- a/scripts/base/frameworks/broker/main.bro +++ b/scripts/base/frameworks/broker/main.bro @@ -14,7 +14,7 @@ export { ## Default address on which to listen. ## ## .. bro:see:: Broker::listen - const default_listen_address = "" &redef; + const default_listen_address = getenv("BRO_DEFAULT_LISTEN_ADDRESS") &redef; ## Default interval to retry connecting to a peer if it cannot be made to work ## initially, or if it ever becomes disconnected. diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index d5bb8f2be9..c20a31279f 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -1,4 +1,4 @@ -@load base/bif/const.bif.bro +@load base/bif/const.bif @load base/bif/types.bif # Type declarations @@ -1797,9 +1797,11 @@ type gtp_delete_pdp_ctx_response_elements: record { }; # Prototypes of Bro built-in functions. -@load base/bif/strings.bif @load base/bif/bro.bif +@load base/bif/stats.bif @load base/bif/reporter.bif +@load base/bif/strings.bif +@load base/bif/option.bif ## Deprecated. This is superseded by the new logging framework. global log_file_name: function(tag: string): string &redef; @@ -4822,17 +4824,3 @@ const global_hash_seed: string = "" &redef; ## files. The larger the value, the more confidence in UID uniqueness. ## The maximum is currently 128 bits. const bits_per_uid: count = 96 &redef; - -# Load these frameworks here because they use fairly deep integration with -# BiFs and script-land defined types. -@load base/frameworks/logging -@load base/frameworks/broker -@load base/frameworks/input -@load base/frameworks/analyzer -@load base/frameworks/files - -@load base/bif - -# Load BiFs defined by plugins. -@load base/bif/plugins - diff --git a/scripts/base/init-frameworks-and-bifs.bro b/scripts/base/init-frameworks-and-bifs.bro new file mode 100644 index 0000000000..f772e2d223 --- /dev/null +++ b/scripts/base/init-frameworks-and-bifs.bro @@ -0,0 +1,15 @@ +# Load these frameworks here because they use fairly deep integration with +# BiFs and script-land defined types. They are also more likely to +# make use of calling BIFs for variable initializations, and that +# can't be done until init-bare.bro has been loaded completely (hence +# the separate file). +@load base/frameworks/logging +@load base/frameworks/broker +@load base/frameworks/input +@load base/frameworks/analyzer +@load base/frameworks/files + +@load base/bif + +# Load BiFs defined by plugins. +@load base/bif/plugins diff --git a/src/input.h b/src/input.h index f0f402b23b..3d0caa459a 100644 --- a/src/input.h +++ b/src/input.h @@ -14,6 +14,7 @@ extern int yydebug; extern int brolex(); extern char last_tok[128]; +extern void add_essential_input_file(const char* file); extern void add_input_file(const char* file); extern void add_input_file_at_front(const char* file); diff --git a/src/main.cc b/src/main.cc index 2277ab0cba..2a61c753b8 100644 --- a/src/main.cc +++ b/src/main.cc @@ -755,7 +755,9 @@ int main(int argc, char** argv) broxygen_mgr = new broxygen::Manager(broxygen_config, bro_argv[0]); - add_input_file("base/init-bare.bro"); + add_essential_input_file("base/init-bare.bro"); + add_essential_input_file("base/init-frameworks-and-bifs.bro"); + if ( ! bare_mode ) add_input_file("base/init-default.bro"); diff --git a/src/scan.l b/src/scan.l index 41fb758bc6..27490c13ad 100644 --- a/src/scan.l +++ b/src/scan.l @@ -821,6 +821,18 @@ void do_atendif() // are referred to (in order to save the locations of tokens and statements, // for error reporting and debugging). static name_list input_files; +static name_list essential_input_files; + +void add_essential_input_file(const char* file) + { + if ( ! file ) + reporter->InternalError("empty filename"); + + if ( ! filename ) + (void) load_files(file); + else + essential_input_files.append(copy_string(file)); + } void add_input_file(const char* file) { @@ -869,7 +881,7 @@ int yywrap() if ( ! did_builtin_init && file_stack.length() == 1 ) { // ### This is a gross hack - we know that the first file - // we parse is bro.init, and after it it's safe to initialize + // we parse is init-bare.bro, and after it it's safe to initialize // the built-ins. Furthermore, we want to initialize the // built-in's *right* after parsing bro.init, so that other // source files can use built-in's when initializing globals. @@ -885,19 +897,22 @@ int yywrap() return 0; // Stack is now empty. - while ( input_files.length() > 0 ) + while ( essential_input_files.length() > 0 || input_files.length() > 0 ) { - if ( load_files(input_files[0]) ) + name_list& files = essential_input_files.length() > 0 ? + essential_input_files : input_files; + + if ( load_files(files[0]) ) { // Don't delete the filename - it's pointed to by // every BroObj created when parsing it. - (void) input_files.remove_nth(0); + (void) files.remove_nth(0); return 0; } // We already scanned the file. Pop it and try the next, // if any. - (void) input_files.remove_nth(0); + (void) files.remove_nth(0); } // For each file scanned so far, and for each @prefix, look for a diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index 768eb520ea..c56e53f28d 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -3,18 +3,21 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2018-03-01-16-07-03 +#open 2018-06-01-20-31-44 #fields name #types string scripts/base/init-bare.bro build/scripts/base/bif/const.bif.bro build/scripts/base/bif/types.bif.bro - build/scripts/base/bif/strings.bif.bro build/scripts/base/bif/bro.bif.bro + build/scripts/base/bif/stats.bif.bro build/scripts/base/bif/reporter.bif.bro + build/scripts/base/bif/strings.bif.bro + build/scripts/base/bif/option.bif.bro build/scripts/base/bif/plugins/Bro_SNMP.types.bif.bro build/scripts/base/bif/plugins/Bro_KRB.types.bif.bro build/scripts/base/bif/event.bif.bro +scripts/base/init-frameworks-and-bifs.bro scripts/base/frameworks/logging/__load__.bro scripts/base/frameworks/logging/main.bro build/scripts/base/bif/logging.bif.bro @@ -52,8 +55,6 @@ scripts/base/init-bare.bro scripts/base/utils/patterns.bro scripts/base/frameworks/files/magic/__load__.bro build/scripts/base/bif/__load__.bro - build/scripts/base/bif/stats.bif.bro - build/scripts/base/bif/option.bif.bro build/scripts/base/bif/broxygen.bif.bro build/scripts/base/bif/pcap.bif.bro build/scripts/base/bif/bloom-filter.bif.bro @@ -175,4 +176,4 @@ scripts/base/init-bare.bro build/scripts/base/bif/plugins/Bro_SQLiteWriter.sqlite.bif.bro scripts/policy/misc/loaded-scripts.bro scripts/base/utils/paths.bro -#close 2018-03-01-16-07-03 +#close 2018-06-01-20-31-44 diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index 5ca6cdd812..11bb9e1fa2 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -3,18 +3,21 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2018-05-17-23-49-59 +#open 2018-06-01-20-31-46 #fields name #types string scripts/base/init-bare.bro build/scripts/base/bif/const.bif.bro build/scripts/base/bif/types.bif.bro - build/scripts/base/bif/strings.bif.bro build/scripts/base/bif/bro.bif.bro + build/scripts/base/bif/stats.bif.bro build/scripts/base/bif/reporter.bif.bro + build/scripts/base/bif/strings.bif.bro + build/scripts/base/bif/option.bif.bro build/scripts/base/bif/plugins/Bro_SNMP.types.bif.bro build/scripts/base/bif/plugins/Bro_KRB.types.bif.bro build/scripts/base/bif/event.bif.bro +scripts/base/init-frameworks-and-bifs.bro scripts/base/frameworks/logging/__load__.bro scripts/base/frameworks/logging/main.bro build/scripts/base/bif/logging.bif.bro @@ -52,8 +55,6 @@ scripts/base/init-bare.bro scripts/base/utils/patterns.bro scripts/base/frameworks/files/magic/__load__.bro build/scripts/base/bif/__load__.bro - build/scripts/base/bif/stats.bif.bro - build/scripts/base/bif/option.bif.bro build/scripts/base/bif/broxygen.bif.bro build/scripts/base/bif/pcap.bif.bro build/scripts/base/bif/bloom-filter.bif.bro @@ -364,4 +365,4 @@ scripts/base/init-default.bro scripts/base/misc/find-filtered-trace.bro scripts/base/misc/version.bro scripts/policy/misc/loaded-scripts.bro -#close 2018-05-17-23-49-59 +#close 2018-06-01-20-31-46 diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index 37a613347c..4a7b2c0854 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -264,7 +264,7 @@ 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -> -0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1525287517.317589, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1527879383.723919, node=bro, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Broker::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Cluster::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Config::LOG)) -> @@ -441,7 +441,7 @@ 0.000000 MetaHookPost CallFunction(Log::create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -> -0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1525287517.317589, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1527879383.723919, node=bro, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(NetControl::check_plugins, , ()) -> 0.000000 MetaHookPost CallFunction(NetControl::init, , ()) -> 0.000000 MetaHookPost CallFunction(Notice::want_pp, , ()) -> @@ -469,6 +469,7 @@ 0.000000 MetaHookPost CallFunction(bro_init, , ()) -> 0.000000 MetaHookPost CallFunction(current_time, , ()) -> 0.000000 MetaHookPost CallFunction(filter_change_tracking, , ()) -> +0.000000 MetaHookPost CallFunction(getenv, , (BRO_DEFAULT_LISTEN_ADDRESS)) -> 0.000000 MetaHookPost CallFunction(getenv, , (CLUSTER_NODE)) -> 0.000000 MetaHookPost CallFunction(global_ids, , ()) -> 0.000000 MetaHookPost CallFunction(network_time, , ()) -> @@ -727,6 +728,7 @@ 0.000000 MetaHookPost LoadFile(0, base<...>/http) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/imap) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/init-default.bro) -> -1 +0.000000 MetaHookPost LoadFile(0, base<...>/init-frameworks-and-bifs.bro) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/input) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/input.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/intel) -> -1 @@ -744,6 +746,7 @@ 0.000000 MetaHookPost LoadFile(0, base<...>/ntlm) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/numbers.bro) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/openflow) -> -1 +0.000000 MetaHookPost LoadFile(0, base<...>/option.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/packet-filter) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/paths.bro) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/patterns.bro) -> -1 @@ -766,6 +769,7 @@ 0.000000 MetaHookPost LoadFile(0, base<...>/software) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/ssh) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/ssl) -> -1 +0.000000 MetaHookPost LoadFile(0, base<...>/stats.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/store.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/strings.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/strings.bro) -> -1 @@ -1062,7 +1066,7 @@ 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1525287517.317589, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1527879383.723919, node=bro, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Broker::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Cluster::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Config::LOG)) @@ -1239,7 +1243,7 @@ 0.000000 MetaHookPre CallFunction(Log::create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) 0.000000 MetaHookPre CallFunction(Log::create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) 0.000000 MetaHookPre CallFunction(Log::create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1525287517.317589, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1527879383.723919, node=bro, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(NetControl::check_plugins, , ()) 0.000000 MetaHookPre CallFunction(NetControl::init, , ()) 0.000000 MetaHookPre CallFunction(Notice::want_pp, , ()) @@ -1267,6 +1271,7 @@ 0.000000 MetaHookPre CallFunction(bro_init, , ()) 0.000000 MetaHookPre CallFunction(current_time, , ()) 0.000000 MetaHookPre CallFunction(filter_change_tracking, , ()) +0.000000 MetaHookPre CallFunction(getenv, , (BRO_DEFAULT_LISTEN_ADDRESS)) 0.000000 MetaHookPre CallFunction(getenv, , (CLUSTER_NODE)) 0.000000 MetaHookPre CallFunction(global_ids, , ()) 0.000000 MetaHookPre CallFunction(network_time, , ()) @@ -1525,6 +1530,7 @@ 0.000000 MetaHookPre LoadFile(0, base<...>/http) 0.000000 MetaHookPre LoadFile(0, base<...>/imap) 0.000000 MetaHookPre LoadFile(0, base<...>/init-default.bro) +0.000000 MetaHookPre LoadFile(0, base<...>/init-frameworks-and-bifs.bro) 0.000000 MetaHookPre LoadFile(0, base<...>/input) 0.000000 MetaHookPre LoadFile(0, base<...>/input.bif.bro) 0.000000 MetaHookPre LoadFile(0, base<...>/intel) @@ -1542,6 +1548,7 @@ 0.000000 MetaHookPre LoadFile(0, base<...>/ntlm) 0.000000 MetaHookPre LoadFile(0, base<...>/numbers.bro) 0.000000 MetaHookPre LoadFile(0, base<...>/openflow) +0.000000 MetaHookPre LoadFile(0, base<...>/option.bif.bro) 0.000000 MetaHookPre LoadFile(0, base<...>/packet-filter) 0.000000 MetaHookPre LoadFile(0, base<...>/paths.bro) 0.000000 MetaHookPre LoadFile(0, base<...>/patterns.bro) @@ -1564,6 +1571,7 @@ 0.000000 MetaHookPre LoadFile(0, base<...>/software) 0.000000 MetaHookPre LoadFile(0, base<...>/ssh) 0.000000 MetaHookPre LoadFile(0, base<...>/ssl) +0.000000 MetaHookPre LoadFile(0, base<...>/stats.bif.bro) 0.000000 MetaHookPre LoadFile(0, base<...>/store.bif.bro) 0.000000 MetaHookPre LoadFile(0, base<...>/strings.bif.bro) 0.000000 MetaHookPre LoadFile(0, base<...>/strings.bro) @@ -1859,7 +1867,7 @@ 0.000000 | HookCallFunction Log::__create_stream(Weird::LOG, [columns=, ev=Weird::log_weird, path=weird]) 0.000000 | HookCallFunction Log::__create_stream(X509::LOG, [columns=, ev=X509::log_x509, path=x509]) 0.000000 | HookCallFunction Log::__create_stream(mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql]) -0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1525287517.317589, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1527879383.723919, node=bro, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction Log::add_default_filter(Broker::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Cluster::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Config::LOG) @@ -2036,7 +2044,7 @@ 0.000000 | HookCallFunction Log::create_stream(Weird::LOG, [columns=, ev=Weird::log_weird, path=weird]) 0.000000 | HookCallFunction Log::create_stream(X509::LOG, [columns=, ev=X509::log_x509, path=x509]) 0.000000 | HookCallFunction Log::create_stream(mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql]) -0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1525287517.317589, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1527879383.723919, node=bro, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction NetControl::check_plugins() 0.000000 | HookCallFunction NetControl::init() 0.000000 | HookCallFunction Notice::want_pp() @@ -2064,6 +2072,7 @@ 0.000000 | HookCallFunction bro_init() 0.000000 | HookCallFunction current_time() 0.000000 | HookCallFunction filter_change_tracking() +0.000000 | HookCallFunction getenv(BRO_DEFAULT_LISTEN_ADDRESS) 0.000000 | HookCallFunction getenv(CLUSTER_NODE) 0.000000 | HookCallFunction global_ids() 0.000000 | HookCallFunction network_time() @@ -2331,6 +2340,7 @@ 0.000000 | HookLoadFile base<...>/http 0.000000 | HookLoadFile base<...>/imap 0.000000 | HookLoadFile base<...>/init-default.bro +0.000000 | HookLoadFile base<...>/init-frameworks-and-bifs.bro 0.000000 | HookLoadFile base<...>/input 0.000000 | HookLoadFile base<...>/input.bif.bro 0.000000 | HookLoadFile base<...>/intel @@ -2348,6 +2358,7 @@ 0.000000 | HookLoadFile base<...>/ntlm 0.000000 | HookLoadFile base<...>/numbers.bro 0.000000 | HookLoadFile base<...>/openflow +0.000000 | HookLoadFile base<...>/option.bif.bro 0.000000 | HookLoadFile base<...>/packet-filter 0.000000 | HookLoadFile base<...>/paths.bro 0.000000 | HookLoadFile base<...>/patterns.bro @@ -2370,6 +2381,7 @@ 0.000000 | HookLoadFile base<...>/software 0.000000 | HookLoadFile base<...>/ssh 0.000000 | HookLoadFile base<...>/ssl +0.000000 | HookLoadFile base<...>/stats.bif.bro 0.000000 | HookLoadFile base<...>/store.bif.bro 0.000000 | HookLoadFile base<...>/strings.bif.bro 0.000000 | HookLoadFile base<...>/strings.bro @@ -2387,7 +2399,7 @@ 0.000000 | HookLoadFile base<...>/x509 0.000000 | HookLoadFile base<...>/xmpp 0.000000 | HookLogInit packet_filter 1/1 {ts (time), node (string), filter (string), init (bool), success (bool)} -0.000000 | HookLogWrite packet_filter [ts=1525287517.317589, node=bro, filter=ip or not ip, init=T, success=T] +0.000000 | HookLogWrite packet_filter [ts=1527879383.723919, node=bro, filter=ip or not ip, init=T, success=T] 0.000000 | HookQueueEvent NetControl::init() 0.000000 | HookQueueEvent bro_init() 0.000000 | HookQueueEvent filter_change_tracking() diff --git a/testing/btest/btest.cfg b/testing/btest/btest.cfg index 8a69f5eddc..81ee5bd5e5 100644 --- a/testing/btest/btest.cfg +++ b/testing/btest/btest.cfg @@ -25,3 +25,4 @@ TMPDIR=%(testbase)s/.tmp BRO_PROFILER_FILE=%(testbase)s/.tmp/script-coverage/XXXXXX BTEST_RST_FILTER=$SCRIPTS/rst-filter BRO_DNS_FAKE=1 +BRO_DEFAULT_LISTEN_ADDRESS=127.0.0.1 From 3a9575bf8a089aa2c8f7cd8dffa569f447a95d4e Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Fri, 1 Jun 2018 13:12:46 -0700 Subject: [PATCH 25/56] Make kerberos initialization static. There does not seem to be any reason to initialize the members more than once globally. --- NEWS | 3 +++ src/analyzer/protocol/krb/KRB.cc | 31 ++++++++++++++++++------------- src/analyzer/protocol/krb/KRB.h | 9 ++++++--- 3 files changed, 27 insertions(+), 16 deletions(-) diff --git a/NEWS b/NEWS index 6f3c047f40..9c2cbcb2e1 100644 --- a/NEWS +++ b/NEWS @@ -242,6 +242,9 @@ New Functionality - Added new SMB events: smb1_transaction_secondary_request, smb1_transaction2_secondary_request, smb1_transaction_response +- Bro can now decrypt Kerberos tickets, and retrieve the authentication from + them, given a suitable keytab file. + Changed Functionality --------------------- diff --git a/src/analyzer/protocol/krb/KRB.cc b/src/analyzer/protocol/krb/KRB.cc index 5a41b0b99b..4ee663dcf1 100644 --- a/src/analyzer/protocol/krb/KRB.cc +++ b/src/analyzer/protocol/krb/KRB.cc @@ -6,13 +6,25 @@ using namespace analyzer::krb; +bool KRB_Analyzer::krb_available = false; +#ifdef USE_KRB5 +krb5_context KRB_Analyzer::krb_context = nullptr; +krb5_keytab KRB_Analyzer::krb_keytab = nullptr; +std::once_flag KRB_Analyzer::krb_initialized; +#endif + KRB_Analyzer::KRB_Analyzer(Connection* conn) - : Analyzer("KRB", conn), - krb_available(false) + : Analyzer("KRB", conn) { interp = new binpac::KRB::KRB_Conn(this); +#ifdef USE_KRB5 + std::call_once(krb_initialized, Initialize_Krb); +#endif + } #ifdef USE_KRB5 +void KRB_Analyzer::Initialize_Krb() + { if ( BifConst::KRB::keytab->Len() == 0 ) return; // no keytab set @@ -37,20 +49,11 @@ KRB_Analyzer::KRB_Analyzer(Connection* conn) return; } krb_available = true; -#endif } +#endif KRB_Analyzer::~KRB_Analyzer() { -#ifdef USE_KRB5 - if ( krb_available ) - { - krb5_error_code retval = krb5_kt_close(krb_context, krb_keytab); - if ( retval ) - reporter->Warning("KRB: Couldn't close keytab (%s)", krb5_get_error_message(krb_context, retval)); - krb5_free_context(krb_context); - } -#endif delete interp; } @@ -125,7 +128,9 @@ StringVal* KRB_Analyzer::GetAuthenticationInfo(const BroString* principal, const StringVal* ret = new StringVal(cp); krb5_free_unparsed_name(krb_context, cp); -#endif return ret; +#else + return nullptr; +#endif } diff --git a/src/analyzer/protocol/krb/KRB.h b/src/analyzer/protocol/krb/KRB.h index 3d924482de..7eee46d838 100644 --- a/src/analyzer/protocol/krb/KRB.h +++ b/src/analyzer/protocol/krb/KRB.h @@ -30,10 +30,13 @@ protected: binpac::KRB::KRB_Conn* interp; - bool krb_available; +private: + static bool krb_available; #ifdef USE_KRB5 - krb5_context krb_context; - krb5_keytab krb_keytab; + static std::once_flag krb_initialized; + static void Initialize_Krb(); + static krb5_context krb_context; + static krb5_keytab krb_keytab; #endif }; From 81d6b4c645a8e42932899ba0044f59f436d00a21 Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Fri, 1 Jun 2018 15:03:41 -0700 Subject: [PATCH 26/56] Updating submodule(s). [nomail] --- aux/binpac | 2 +- aux/bro-aux | 2 +- aux/broccoli | 2 +- aux/broctl | 2 +- aux/broker | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/aux/binpac b/aux/binpac index 0b2ef114fd..951aeae8e4 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit 0b2ef114fdac4c135d357693d7e74a441dee8db3 +Subproject commit 951aeae8e4a08c598203cf61387f015ec4e0849d diff --git a/aux/bro-aux b/aux/bro-aux index ad99dc534f..eeb677ff69 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit ad99dc534f2574a47a808d677fc76098f42a1b54 +Subproject commit eeb677ff696f8ea3eaa43a765fe40da07ed5281d diff --git a/aux/broccoli b/aux/broccoli index 701a539f29..d9041cc95d 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit 701a539f295f138bb1c44953310e083a4210fe1b +Subproject commit d9041cc95d2232dbbcf36647f34537da22e360ff diff --git a/aux/broctl b/aux/broctl index 7e68ad436e..fc7abc2c5e 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 7e68ad436e122fa95c87b6caca0e2e7b20dd5b97 +Subproject commit fc7abc2c5e459b51d60b2036db428053b5fb27f5 diff --git a/aux/broker b/aux/broker index 7b84848bde..da4f84a4cf 160000 --- a/aux/broker +++ b/aux/broker @@ -1 +1 @@ -Subproject commit 7b84848bded443637fa34e76f7d8558bd1cafbee +Subproject commit da4f84a4cf9921298910e5b61214f32de27c632d From 51f20136d5ce8a768ec77ada34b4bf326933d11c Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Sat, 2 Jun 2018 04:37:08 -0400 Subject: [PATCH 27/56] Remove some UTF-8 characters that snuck into a few strings. --- scripts/base/files/pe/consts.bro | 2 +- .../base/protocols/smb/const-nt-status.bro | 20 +++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/scripts/base/files/pe/consts.bro b/scripts/base/files/pe/consts.bro index 35ad9c3c61..3dcfddec79 100644 --- a/scripts/base/files/pe/consts.bro +++ b/scripts/base/files/pe/consts.bro @@ -65,7 +65,7 @@ export { [9] = "WINDOWS_CE_GUI", [10] = "EFI_APPLICATION", [11] = "EFI_BOOT_SERVICE_DRIVER", - [12] = "EFI_RUNTIME_
DRIVER", + [12] = "EFI_RUNTIME_DRIVER", [13] = "EFI_ROM", [14] = "XBOX" } &default=function(i: count):string { return fmt("unknown-%d", i); }; diff --git a/scripts/base/protocols/smb/const-nt-status.bro b/scripts/base/protocols/smb/const-nt-status.bro index 8804522ed9..ea7c7fe9ed 100644 --- a/scripts/base/protocols/smb/const-nt-status.bro +++ b/scripts/base/protocols/smb/const-nt-status.bro @@ -64,8 +64,8 @@ redef SMB::statuses += { [0x40000007] = [$id="BAD_CURRENT_DIRECTORY", $desc="{Invalid Current Directory} The process cannot switch to the startup current directory %hs. Select OK to set the current directory to %hs, or select CANCEL to exit."], [0x40000008] = [$id="SERIAL_MORE_WRITES", $desc="{Serial IOCTL Complete} A serial I/O operation was completed by another write to a serial port. (The IOCTL_SERIAL_XOFF_COUNTER reached zero.)"], [0x40000009] = [$id="REGISTRY_RECOVERED", $desc="{Registry Recovery} One of the files that contains the system registry data had to be recovered by using a log or alternate copy. The recovery was successful."], - [0x4000000A] = [$id="FT_READ_RECOVERY_FROM_BACKUP", $desc="{Redundant Read} To satisfy a read request, the Windows NT fault-tolerant file system successfully read the requested data from a redundant copy. This was done because the file system encountered a failure on a member of the fault-tolerant volume but was unable to reassign the failing area of the device."], - [0x4000000B] = [$id="FT_WRITE_RECOVERY", $desc="{Redundant Write} To satisfy a write request, the Windows NT fault-tolerant file system successfully wrote a redundant copy of the information. This was done because the file system encountered a failure on a member of the fault-tolerant volume but was unable to reassign the failing area of the device."], + [0x4000000A] = [$id="FT_READ_RECOVERY_FROM_BACKUP", $desc="{Redundant Read} To satisfy a read request, the Windows NT fault-tolerant file system successfully read the requested data from a redundant copy. This was done because the file system encountered a failure on a member of the fault-tolerant volume but was unable to reassign the failing area of the device."], + [0x4000000B] = [$id="FT_WRITE_RECOVERY", $desc="{Redundant Write} To satisfy a write request, the Windows NT fault-tolerant file system successfully wrote a redundant copy of the information. This was done because the file system encountered a failure on a member of the fault-tolerant volume but was unable to reassign the failing area of the device."], [0x4000000C] = [$id="SERIAL_COUNTER_TIMEOUT", $desc="{Serial IOCTL Timeout} A serial I/O operation completed because the time-out period expired. (The IOCTL_SERIAL_XOFF_COUNTER had not reached zero.)"], [0x4000000D] = [$id="NULL_LM_PASSWORD", $desc="{Password Too Complex} The Windows password is too complex to be converted to a LAN Manager password. The LAN Manager password that returned is a NULL string."], [0x4000000E] = [$id="IMAGE_MACHINE_TYPE_MISMATCH", $desc="{Machine Type Mismatch} The image file %hs is valid but is for a machine type other than the current machine. Select OK to continue, or CANCEL to fail the DLL load."], @@ -839,12 +839,12 @@ redef SMB::statuses += { [0xC0000303] = [$id="WMI_ALREADY_ENABLED", $desc="Collection or events for the WMI GUID is already enabled."], [0xC0000304] = [$id="MFT_TOO_FRAGMENTED", $desc="The master file table on the volume is too fragmented to complete this operation."], [0xC0000305] = [$id="COPY_PROTECTION_FAILURE", $desc="Copy protection failure."], - [0xC0000306] = [$id="CSS_AUTHENTICATION_FAILURE", $desc="Copy protection error—DVD CSS Authentication failed."], - [0xC0000307] = [$id="CSS_KEY_NOT_PRESENT", $desc="Copy protection error—The specified sector does not contain a valid key."], - [0xC0000308] = [$id="CSS_KEY_NOT_ESTABLISHED", $desc="Copy protection error—DVD session key not established."], - [0xC0000309] = [$id="CSS_SCRAMBLED_SECTOR", $desc="Copy protection error—The read failed because the sector is encrypted."], - [0xC000030A] = [$id="CSS_REGION_MISMATCH", $desc="Copy protection error—The region of the specified DVD does not correspond to the region setting of the drive."], - [0xC000030B] = [$id="CSS_RESETS_EXHAUSTED", $desc="Copy protection error—The region setting of the drive may be permanent."], + [0xC0000306] = [$id="CSS_AUTHENTICATION_FAILURE", $desc="Copy protection error-DVD CSS Authentication failed."], + [0xC0000307] = [$id="CSS_KEY_NOT_PRESENT", $desc="Copy protection error-The specified sector does not contain a valid key."], + [0xC0000308] = [$id="CSS_KEY_NOT_ESTABLISHED", $desc="Copy protection error-DVD session key not established."], + [0xC0000309] = [$id="CSS_SCRAMBLED_SECTOR", $desc="Copy protection error-The read failed because the sector is encrypted."], + [0xC000030A] = [$id="CSS_REGION_MISMATCH", $desc="Copy protection error-The region of the specified DVD does not correspond to the region setting of the drive."], + [0xC000030B] = [$id="CSS_RESETS_EXHAUSTED", $desc="Copy protection error-The region setting of the drive may be permanent."], [0xC0000320] = [$id="PKINIT_FAILURE", $desc="The Kerberos protocol encountered an error while validating the KDC certificate during smart card logon. There is more information in the system event log."], [0xC0000321] = [$id="SMARTCARD_SUBSYSTEM_FAILURE", $desc="The Kerberos protocol encountered an error while attempting to use the smart card subsystem."], [0xC0000322] = [$id="NO_KERB_KEY", $desc="The target server does not have acceptable Kerberos credentials."], @@ -855,7 +855,7 @@ redef SMB::statuses += { [0xC0000354] = [$id="DEBUGGER_INACTIVE", $desc="An attempt to do an operation on a debug port failed because the port is in the process of being deleted."], [0xC0000355] = [$id="DS_VERSION_CHECK_FAILURE", $desc="This version of Windows is not compatible with the behavior version of the directory forest, domain, or domain controller."], [0xC0000356] = [$id="AUDITING_DISABLED", $desc="The specified event is currently not being audited."], - [0xC0000357] = [$id="PRENT4_MACHINE_ACCOUNT", $desc="The machine account was created prior to Windows NT 4.0. The account needs to be recreated."], + [0xC0000357] = [$id="PRENT4_MACHINE_ACCOUNT", $desc="The machine account was created prior to Windows NT 4.0. The account needs to be recreated."], [0xC0000358] = [$id="DS_AG_CANT_HAVE_UNIVERSAL_MEMBER", $desc="An account group cannot have a universal group as a member."], [0xC0000359] = [$id="INVALID_IMAGE_WIN_32", $desc="The specified image file did not have the correct format; it appears to be a 32-bit Windows image."], [0xC000035A] = [$id="INVALID_IMAGE_WIN_64", $desc="The specified image file did not have the correct format; it appears to be a 64-bit Windows image."], @@ -1790,4 +1790,4 @@ redef SMB::statuses += { [0xC03A0017] = [$id="VHD_CHILD_PARENT_SIZE_MISMATCH", $desc="The chain of virtual hard disks is corrupted. There is a mismatch in the virtual sizes of the parent virtual hard disk and differencing disk."], [0xC03A0018] = [$id="VHD_DIFFERENCING_CHAIN_CYCLE_DETECTED", $desc="The chain of virtual hard disks is corrupted. A differencing disk is indicated in its own parent chain."], [0xC03A0019] = [$id="VHD_DIFFERENCING_CHAIN_ERROR_IN_PARENT", $desc="The chain of virtual hard disks is inaccessible. There was an error opening a virtual hard disk further up the chain."], -}; \ No newline at end of file +}; From cd18d96205851aa9b81bcb8f0c6960768b457f72 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Sat, 2 Jun 2018 04:57:48 -0400 Subject: [PATCH 28/56] Removed a few more discovered UTF-8 characters in Bro scripts. --- scripts/base/protocols/smb/const-nt-status.bro | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/base/protocols/smb/const-nt-status.bro b/scripts/base/protocols/smb/const-nt-status.bro index ea7c7fe9ed..f985e72a3b 100644 --- a/scripts/base/protocols/smb/const-nt-status.bro +++ b/scripts/base/protocols/smb/const-nt-status.bro @@ -494,7 +494,7 @@ redef SMB::statuses += { [0xC0000131] = [$id="INVALID_IMAGE_WIN_16", $desc="The specified image file did not have the correct format: it appears to be a 16-bit Windows image."], [0xC0000132] = [$id="LOGON_SERVER_CONFLICT", $desc="The Netlogon service cannot start because another Netlogon service running in the domain conflicts with the specified role."], [0xC0000133] = [$id="TIME_DIFFERENCE_AT_DC", $desc="The time at the primary domain controller is different from the time at the backup domain controller or member server by too large an amount."], - [0xC0000134] = [$id="SYNCHRONIZATION_REQUIRED", $desc="The SAM database on a Windows Server is significantly out of synchronization with the copy on the domain controller. A complete synchronization is required."], + [0xC0000134] = [$id="SYNCHRONIZATION_REQUIRED", $desc="The SAM database on a Windows Server is significantly out of synchronization with the copy on the domain controller. A complete synchronization is required."], [0xC0000135] = [$id="DLL_NOT_FOUND", $desc="{Unable To Locate Component} This application has failed to start because %hs was not found. Reinstalling the application may fix this problem."], [0xC0000136] = [$id="OPEN_FAILED", $desc="The NtCreateFile API failed. This error should never be returned to an application; it is a place holder for the Windows LAN Manager Redirector to use in its internal error-mapping routines."], [0xC0000137] = [$id="IO_PRIVILEGE_FAILED", $desc="{Privilege Failed} The I/O permissions for the process could not be changed."], @@ -536,7 +536,7 @@ redef SMB::statuses += { [0xC000015B] = [$id="LOGON_TYPE_NOT_GRANTED", $desc="A user has requested a type of logon (for example, interactive or network) that has not been granted. An administrator has control over who may logon interactively and through the network."], [0xC000015C] = [$id="NOT_REGISTRY_FILE", $desc="The system has attempted to load or restore a file into the registry, and the specified file is not in the format of a registry file."], [0xC000015D] = [$id="NT_CROSS_ENCRYPTION_REQUIRED", $desc="An attempt was made to change a user password in the security account manager without providing the necessary Windows cross-encrypted password."], - [0xC000015E] = [$id="DOMAIN_CTRLR_CONFIG_ERROR", $desc="A Windows Server has an incorrect configuration."], + [0xC000015E] = [$id="DOMAIN_CTRLR_CONFIG_ERROR", $desc="A Windows Server has an incorrect configuration."], [0xC000015F] = [$id="FT_MISSING_MEMBER", $desc="An attempt was made to explicitly access the secondary copy of information via a device control to the fault tolerance driver and the secondary copy is not present in the system."], [0xC0000160] = [$id="ILL_FORMED_SERVICE_ENTRY", $desc="A configuration registry node that represents a driver service entry was ill-formed and did not contain the required value entries."], [0xC0000161] = [$id="ILLEGAL_CHARACTER", $desc="An illegal character was encountered. For a multibyte character set, this includes a lead byte without a succeeding trail byte. For the Unicode character set this includes the characters 0xFFFF and 0xFFFE."], @@ -577,7 +577,7 @@ redef SMB::statuses += { [0xC0000188] = [$id="LOG_FILE_FULL", $desc="The log file space is insufficient to support this operation."], [0xC0000189] = [$id="TOO_LATE", $desc="A write operation was attempted to a volume after it was dismounted."], [0xC000018A] = [$id="NO_TRUST_LSA_SECRET", $desc="The workstation does not have a trust secret for the primary domain in the local LSA database."], - [0xC000018B] = [$id="NO_TRUST_SAM_ACCOUNT", $desc="The SAM database on the Windows Server does not have a computer account for this workstation trust relationship."], + [0xC000018B] = [$id="NO_TRUST_SAM_ACCOUNT", $desc="The SAM database on the Windows Server does not have a computer account for this workstation trust relationship."], [0xC000018C] = [$id="TRUSTED_DOMAIN_FAILURE", $desc="The logon request failed because the trust relationship between the primary domain and the trusted domain failed."], [0xC000018D] = [$id="TRUSTED_RELATIONSHIP_FAILURE", $desc="The logon request failed because the trust relationship between this workstation and the primary domain failed."], [0xC000018E] = [$id="EVENTLOG_FILE_CORRUPT", $desc="The Eventlog log file is corrupt."], @@ -833,7 +833,7 @@ redef SMB::statuses += { [0xC00002FD] = [$id="KDC_UNKNOWN_ETYPE", $desc="The encryption type requested is not supported by the KDC."], [0xC00002FE] = [$id="SHUTDOWN_IN_PROGRESS", $desc="A system shutdown is in progress."], [0xC00002FF] = [$id="SERVER_SHUTDOWN_IN_PROGRESS", $desc="The server machine is shutting down."], - [0xC0000300] = [$id="NOT_SUPPORTED_ON_SBS", $desc="This operation is not supported on a computer running Windows Server 2003 for Small Business Server."], + [0xC0000300] = [$id="NOT_SUPPORTED_ON_SBS", $desc="This operation is not supported on a computer running Windows Server 2003 for Small Business Server."], [0xC0000301] = [$id="WMI_GUID_DISCONNECTED", $desc="The WMI GUID is no longer available."], [0xC0000302] = [$id="WMI_ALREADY_DISABLED", $desc="Collection or events for the WMI GUID is already disabled."], [0xC0000303] = [$id="WMI_ALREADY_ENABLED", $desc="Collection or events for the WMI GUID is already enabled."], From fde88fa7177f7432d8611346926b428ff6dfc628 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 2 Jun 2018 17:29:01 -0500 Subject: [PATCH 29/56] #120: Pull in PR, and make a couple small cleanup tweaks. --- src/analyzer/protocol/ssh/ssh-analyzer.pac | 2 +- src/analyzer/protocol/ssh/ssh-protocol.pac | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/analyzer/protocol/ssh/ssh-analyzer.pac b/src/analyzer/protocol/ssh/ssh-analyzer.pac index bdb2553849..0ee0b92569 100644 --- a/src/analyzer/protocol/ssh/ssh-analyzer.pac +++ b/src/analyzer/protocol/ssh/ssh-analyzer.pac @@ -178,7 +178,7 @@ refine flow SSH_Flow += { %{ switch (v) { case SSH1: - return packet_length + 4 + 8 -(packet_length%8); + return packet_length + 4 + 8 - (packet_length % 8); case SSH2: return packet_length + 4; default: diff --git a/src/analyzer/protocol/ssh/ssh-protocol.pac b/src/analyzer/protocol/ssh/ssh-protocol.pac index a86e6cef8e..bf09f6e168 100644 --- a/src/analyzer/protocol/ssh/ssh-protocol.pac +++ b/src/analyzer/protocol/ssh/ssh-protocol.pac @@ -38,7 +38,7 @@ type SSH1_Key_Exchange(is_orig: bool, packet_length: uint32) = record { msg_type : uint8; message : SSH1_Message(is_orig, msg_type, packet_length - 5); crc : uint32; -} &length = packet_length + 8 - (packet_length % 8); +} &length = $context.flow.get_kex_length($context.connection.get_version(), packet_length) - 4; type SSH1_Message(is_orig: bool, msg_type: uint8, length: uint32) = case msg_type of { SSH_SMSG_PUBLIC_KEY -> public_key : SSH1_PUBLIC_KEY(length); From 19b893a5bcd8ff90d0b898fd39ceefe7fd885f6d Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 4 Jun 2018 13:52:46 -0500 Subject: [PATCH 30/56] Make 0 be a valid packet source timestamp For fuzzed/damaged/corrupted pcaps, a timestamp of 0 could lead to an infinite loop in Bro as it interprets that as meaning the packet source is not ready yet. --- CHANGES | 4 ++++ VERSION | 2 +- src/Serializer.cc | 4 ++-- src/iosource/Manager.cc | 4 ++-- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/CHANGES b/CHANGES index e807bb0525..52bc4656a2 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.5-642 | 2018-06-04 13:52:46 -0500 + + * Make 0 be a valid packet source timestamp (Corelight) + 2.5-641 | 2018-06-04 09:18:59 -0700 * Add Broker::publish_and_relay BIF diff --git a/VERSION b/VERSION index 15ce2d3aad..6de87095f8 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.5-641 +2.5-642 diff --git a/src/Serializer.cc b/src/Serializer.cc index b759334b4d..0366c36c81 100644 --- a/src/Serializer.cc +++ b/src/Serializer.cc @@ -1017,7 +1017,7 @@ double EventPlayer::NextTimestamp(double* local_network_time) return ne_time; if ( ! io ) - return 0; + return -1; // Read next event if we don't have one waiting. if ( ! ne_time ) @@ -1028,7 +1028,7 @@ double EventPlayer::NextTimestamp(double* local_network_time) } if ( ! ne_time ) - return 0; + return -1; if ( ! network_time ) { diff --git a/src/iosource/Manager.cc b/src/iosource/Manager.cc index 80fa5fe860..390449da81 100644 --- a/src/iosource/Manager.cc +++ b/src/iosource/Manager.cc @@ -81,7 +81,7 @@ IOSource* Manager::FindSoonest(double* ts) all_idle = false; double local_network_time = 0; double ts = (*i)->src->NextTimestamp(&local_network_time); - if ( ts > 0 && ts < soonest_ts ) + if ( ts >= 0 && ts < soonest_ts ) { soonest_ts = ts; soonest_src = (*i)->src; @@ -162,7 +162,7 @@ IOSource* Manager::FindSoonest(double* ts) { double local_network_time = 0; double ts = src->src->NextTimestamp(&local_network_time); - if ( ts > 0.0 && ts < soonest_ts ) + if ( ts >= 0.0 && ts < soonest_ts ) { soonest_ts = ts; soonest_src = src->src; From a89151776242773a89ea1c8557e28d2b08c31e61 Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Mon, 4 Jun 2018 11:43:20 -0700 Subject: [PATCH 31/56] Fix read at invalid address in X509 extension parser. When encountering an extension unknown to OpenSSL, we would read from the wrong memory location. Also added a testcase to prevent this specific case from happening again. --- src/file_analysis/analyzer/x509/X509Common.cc | 2 +- .../.stdout | 4 ++++ .../base/protocols/ssl/x509-invalid-extension.test | 11 +++++++++++ 3 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 testing/btest/Baseline/scripts.base.protocols.ssl.x509-invalid-extension/.stdout create mode 100644 testing/btest/scripts/base/protocols/ssl/x509-invalid-extension.test diff --git a/src/file_analysis/analyzer/x509/X509Common.cc b/src/file_analysis/analyzer/x509/X509Common.cc index b101f502ff..38102ed97e 100644 --- a/src/file_analysis/analyzer/x509/X509Common.cc +++ b/src/file_analysis/analyzer/x509/X509Common.cc @@ -244,7 +244,7 @@ void file_analysis::X509Common::ParseExtension(X509_EXTENSION* ex, EventHandlerP int len = i2d_ASN1_OCTET_STRING(X509_EXTENSION_get_data(ex), &buf); if ( len >=0 ) { - BIO_write(bio, &buf, len); + BIO_write(bio, buf, len); OPENSSL_free(buf); } } diff --git a/testing/btest/Baseline/scripts.base.protocols.ssl.x509-invalid-extension/.stdout b/testing/btest/Baseline/scripts.base.protocols.ssl.x509-invalid-extension/.stdout new file mode 100644 index 0000000000..a56a7a6080 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.ssl.x509-invalid-extension/.stdout @@ -0,0 +1,4 @@ +UNDEF +\x04a0_\xa1]\xa0[0Y0W0U\x16\x09image/gif0!0\x1f0\x07\x06\x05+\x0e\x03\x02\x1a\x04\x14\x8f\xe5\xd3\x1a\x86\xac\x8d\x8ek\xc3\xcf\x80j\xd4H\x18,{\x19.0%\x16#http://logo.verisign.com/vslogo.gif +UNDEF +\x04a0_\xa1]\xa0[0Y0W0U\x16\x09image/gif0!0\x1f0\x07\x06\x05+\x0e\x03\x02\x1a\x04\x14\x8f\xe5\xd3\x1a\x86\xac\x8d\x8ek\xc3\xcf\x80j\xd4H\x18,{\x19.0%\x16#http://logo.verisign.com/vslogo.gif diff --git a/testing/btest/scripts/base/protocols/ssl/x509-invalid-extension.test b/testing/btest/scripts/base/protocols/ssl/x509-invalid-extension.test new file mode 100644 index 0000000000..de0dc9e59f --- /dev/null +++ b/testing/btest/scripts/base/protocols/ssl/x509-invalid-extension.test @@ -0,0 +1,11 @@ +# @TEST-EXEC: bro -C -r $TRACES/tls/ocsp-stapling.trace %INPUT +# @TEST-EXEC: btest-diff .stdout + +event x509_extension(f: fa_file, ext: X509::Extension) + { + if ( ext$oid != "1.3.6.1.5.5.7.1.12" ) + return; + + print ext$short_name; + print ext$value; + } From b5b688b5f546b94b822e8fae88d384089880c7af Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 5 Jun 2018 15:19:16 -0500 Subject: [PATCH 32/56] Update `make doc`: don't copy broker docs --- CHANGES | 4 ++++ VERSION | 2 +- doc/CMakeLists.txt | 5 ----- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/CHANGES b/CHANGES index e928add3da..de7e57cc94 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.5-647 | 2018-06-05 15:19:16 -0500 + + * Update `make doc`: don't copy broker docs (Corelight) + 2.5-646 | 2018-06-05 11:31:43 -0500 * Add NCP::max_frame_size tuning option (Corelight) diff --git a/VERSION b/VERSION index a0572ad9b1..e4a6c3fab3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.5-646 +2.5-647 diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index 2563375dcc..0edf2429ab 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -4,8 +4,6 @@ set(BROXYGEN_SCRIPT_OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/broxygen_script_output) set(BROXYGEN_CACHE_DIR ${CMAKE_CURRENT_BINARY_DIR}/broxygen_cache) set(BROCCOLI_DOCS_SRC ${CMAKE_BINARY_DIR}/aux/broccoli/doc/html) set(BROCCOLI_DOCS_DST ${CMAKE_BINARY_DIR}/html/broccoli-api) -set(BROKER_DOCS_SRC ${CMAKE_BINARY_DIR}/aux/broker/doc/html) -set(BROKER_DOCS_DST ${CMAKE_BINARY_DIR}/html/broker-manual) # Find out what BROPATH to use when executing bro. execute_process(COMMAND ${CMAKE_BINARY_DIR}/bro-path-dev @@ -63,9 +61,6 @@ add_custom_target(sphinxdoc COMMAND "${CMAKE_COMMAND}" -E create_symlink ${SPHINX_OUTPUT_DIR}/html ${CMAKE_BINARY_DIR}/html - # Copy Broker manual into output dir. - COMMAND rm -rf ${BROKER_DOCS_DST} && - cp -r ${BROKER_DOCS_SRC} ${BROKER_DOCS_DST} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} COMMENT "[Sphinx] Generate HTML documentation in ${CMAKE_BINARY_DIR}/html") From 0b4871daf4d60ee1d8e414be8152a4e7fb972b9b Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 5 Jun 2018 17:32:47 -0500 Subject: [PATCH 33/56] BIT-1936: improve Broxygen warnings --- CHANGES | 4 ++++ VERSION | 2 +- src/broxygen/Manager.cc | 35 ++++++++++++++++++++++++----------- 3 files changed, 29 insertions(+), 12 deletions(-) diff --git a/CHANGES b/CHANGES index de7e57cc94..c059e1ce7f 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.5-648 | 2018-06-05 17:32:47 -0500 + + * BIT-1936: improve Broxygen warnings (Corelight) + 2.5-647 | 2018-06-05 15:19:16 -0500 * Update `make doc`: don't copy broker docs (Corelight) diff --git a/VERSION b/VERSION index e4a6c3fab3..e458614976 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.5-647 +2.5-648 diff --git a/src/broxygen/Manager.cc b/src/broxygen/Manager.cc index 9e33e6919b..4fd28d60f5 100644 --- a/src/broxygen/Manager.cc +++ b/src/broxygen/Manager.cc @@ -12,7 +12,13 @@ using namespace std; static void DbgAndWarn(const char* msg) { - reporter->InternalWarning("%s", msg); + if ( reporter->Errors() ) + // We've likely already reported to real source of the problem + // as an error, avoid adding an additional warning which may + // be confusing. + return; + + reporter->Warning("%s", msg); DBG_LOG(DBG_BROXYGEN, "%s", msg); } @@ -22,7 +28,8 @@ static void WarnMissingScript(const char* type, const ID* id, if ( script == "" ) return; - DbgAndWarn(fmt("Can't document %s %s, lookup of %s failed", + DbgAndWarn(fmt("Can't generate Broxygen doumentation for %s %s, " + "lookup of %s failed", type, id->Name(), script.c_str())); } @@ -122,7 +129,8 @@ void Manager::Script(const string& path) if ( scripts.GetInfo(name) ) { - DbgAndWarn(fmt("Duplicate script documentation: %s", name.c_str())); + DbgAndWarn(fmt("Duplicate Broxygen script documentation: %s", + name.c_str())); return; } @@ -138,7 +146,8 @@ void Manager::Script(const string& path) if ( packages.GetInfo(name) ) { - DbgAndWarn(fmt("Duplicate package documentation: %s", name.c_str())); + DbgAndWarn(fmt("Duplicate Broxygen package documentation: %s", + name.c_str())); return; } @@ -155,7 +164,8 @@ void Manager::ScriptDependency(const string& path, const string& dep) if ( dep.empty() ) { - DbgAndWarn(fmt("Empty script doc dependency: %s", path.c_str())); + DbgAndWarn(fmt("Empty Broxygen script doc dependency: %s", + path.c_str())); return; } @@ -165,8 +175,8 @@ void Manager::ScriptDependency(const string& path, const string& dep) if ( ! script_info ) { - DbgAndWarn(fmt("Failed to add script doc dependency %s for %s", - depname.c_str(), name.c_str())); + DbgAndWarn(fmt("Failed to add Broxygen script doc dependency %s " + "for %s", depname.c_str(), name.c_str())); return; } @@ -189,7 +199,7 @@ void Manager::ModuleUsage(const string& path, const string& module) if ( ! script_info ) { - DbgAndWarn(fmt("Failed to add module usage %s in %s", + DbgAndWarn(fmt("Failed to add Broxygen module usage %s in %s", module.c_str(), name.c_str())); return; } @@ -231,7 +241,8 @@ void Manager::StartType(ID* id) if ( id->GetLocationInfo() == &no_location ) { - DbgAndWarn(fmt("Can't document %s, no location available", id->Name())); + DbgAndWarn(fmt("Can't generate broxygen doumentation for %s, " + "no location available", id->Name())); return; } @@ -323,7 +334,8 @@ void Manager::RecordField(const ID* id, const TypeDecl* field, if ( ! idd ) { - DbgAndWarn(fmt("Can't document record field %s, unknown record: %s", + DbgAndWarn(fmt("Can't generate broxygen doumentation for " + "record field %s, unknown record: %s", field->id, id->Name())); return; } @@ -348,7 +360,8 @@ void Manager::Redef(const ID* id, const string& path) if ( ! id_info ) { - DbgAndWarn(fmt("Can't document redef of %s, identifier lookup failed", + DbgAndWarn(fmt("Can't generate broxygen doumentation for " + "redef of %s, identifier lookup failed", id->Name())); return; } From 10fdb16f3ba678032778032bcbebb036676107e8 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 6 Jun 2018 14:02:22 -0500 Subject: [PATCH 34/56] Updating submodule(s). [nomail] --- aux/broctl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/broctl b/aux/broctl index fc7abc2c5e..64fc253007 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit fc7abc2c5e459b51d60b2036db428053b5fb27f5 +Subproject commit 64fc25300723d753217b5cf5e8bee2d4500409e1 From 9822fc252d5e92208704df4a388ea31989869499 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 6 Jun 2018 16:20:18 -0500 Subject: [PATCH 35/56] Improve Broker performance Now manually keeps track of peer count instead of querying Broker for that information (which would result in waiting upon a blocking request to the core actor). --- CHANGES | 4 ++++ VERSION | 2 +- src/broker/Manager.cc | 30 +++++++++++++++--------------- src/broker/Manager.h | 2 ++ 4 files changed, 22 insertions(+), 16 deletions(-) diff --git a/CHANGES b/CHANGES index c059e1ce7f..a39792e601 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.5-650 | 2018-06-06 16:20:18 -0500 + + * Improve Broker performance (Corelight) + 2.5-648 | 2018-06-05 17:32:47 -0500 * BIT-1936: improve Broxygen warnings (Corelight) diff --git a/VERSION b/VERSION index e458614976..1b47653967 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.5-648 +2.5-650 diff --git a/src/broker/Manager.cc b/src/broker/Manager.cc index 8184a86111..3d666fa1d4 100644 --- a/src/broker/Manager.cc +++ b/src/broker/Manager.cc @@ -136,6 +136,7 @@ Manager::BrokerState::BrokerState(broker::broker_options options) Manager::Manager(bool reading_pcaps) { bound_port = 0; + peer_count = 0; next_timestamp = 1; SetIdle(false); @@ -205,7 +206,7 @@ bool Manager::Active() if ( bound_port > 0 ) return true; - return bstate->endpoint.peers().size(); + return peer_count > 0; } void Manager::AdvanceTime(double seconds_since_unix_epoch) @@ -301,7 +302,7 @@ bool Manager::PublishEvent(string topic, std::string name, broker::vector args) if ( bstate->endpoint.is_shutdown() ) return true; - if ( ! bstate->endpoint.peers().size() ) + if ( peer_count == 0 ) return true; DBG_LOG(DBG_BROKER, "Publishing event: %s", @@ -317,7 +318,7 @@ bool Manager::PublishEvent(string topic, RecordVal* args) if ( bstate->endpoint.is_shutdown() ) return true; - if ( ! bstate->endpoint.peers().size() ) + if ( peer_count == 0 ) return true; if ( ! args->Lookup(0) ) @@ -347,7 +348,7 @@ bool Manager::RelayEvent(std::string first_topic, if ( bstate->endpoint.is_shutdown() ) return true; - if ( ! bstate->endpoint.peers().size() ) + if ( peer_count == 0 ) return true; DBG_LOG(DBG_BROKER, "Publishing %s-relay event: %s", @@ -381,7 +382,7 @@ bool Manager::RelayEvent(std::string first_topic, if ( bstate->endpoint.is_shutdown() ) return true; - if ( ! bstate->endpoint.peers().size() ) + if ( peer_count == 0 ) return true; if ( ! args->Lookup(0) ) @@ -413,7 +414,7 @@ bool Manager::PublishIdentifier(std::string topic, std::string id) if ( bstate->endpoint.is_shutdown() ) return true; - if ( ! bstate->endpoint.peers().size() ) + if ( peer_count == 0 ) return true; ID* i = global_scope()->Lookup(id.c_str()); @@ -453,7 +454,7 @@ bool Manager::PublishLogCreate(EnumVal* stream, EnumVal* writer, if ( bstate->endpoint.is_shutdown() ) return true; - if ( ! bstate->endpoint.peers().size() ) + if ( peer_count == 0 ) return true; auto stream_id = stream->Type()->AsEnumType()->Lookup(stream->AsEnum()); @@ -507,7 +508,7 @@ bool Manager::PublishLogWrite(EnumVal* stream, EnumVal* writer, string path, int if ( bstate->endpoint.is_shutdown() ) return true; - if ( ! bstate->endpoint.peers().size() ) + if ( peer_count == 0 ) return true; auto stream_id_num = stream->AsEnum(); @@ -1185,16 +1186,19 @@ void Manager::ProcessStatus(broker::status stat) break; case broker::sc::peer_added: - assert(ctx); - log_mgr->SendAllWritersTo(*ctx); + ++peer_count; + assert(ctx); + log_mgr->SendAllWritersTo(*ctx); event = Broker::peer_added; break; case broker::sc::peer_removed: + --peer_count; event = Broker::peer_removed; break; case broker::sc::peer_lost: + --peer_count; event = Broker::peer_lost; break; } @@ -1490,11 +1494,7 @@ bool Manager::TrackStoreQuery(StoreHandleVal* handle, broker::request_id id, const Stats& Manager::GetStatistics() { - if ( bstate->endpoint.is_shutdown() ) - statistics.num_peers = 0; - else - statistics.num_peers = bstate->endpoint.peers().size(); - + statistics.num_peers = peer_count; statistics.num_stores = data_stores.size(); statistics.num_pending_queries = pending_queries.size(); diff --git a/src/broker/Manager.h b/src/broker/Manager.h index 976aa82a03..182203f829 100644 --- a/src/broker/Manager.h +++ b/src/broker/Manager.h @@ -414,6 +414,8 @@ private: Stats statistics; double next_timestamp; bool reading_pcaps; + int peer_count; + static int script_scope; static VectorType* vector_of_data_type; From f50effd364ab315855944b947a32b3b1f4ef7ee8 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 7 Jun 2018 09:51:46 -0500 Subject: [PATCH 36/56] Fix signed/unsigned comparison compiler warning The signed int in this case is essentially a constant anyway. --- CHANGES | 4 ++++ VERSION | 2 +- src/analyzer/protocol/ncp/NCP.cc | 2 +- src/analyzer/protocol/ncp/NCP.h | 4 ++-- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/CHANGES b/CHANGES index a39792e601..188dd6439c 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.5-651 | 2018-06-07 09:57:29 -0500 + + * Fix signed/unsigned comparison compiler warning (Corelight) + 2.5-650 | 2018-06-06 16:20:18 -0500 * Improve Broker performance (Corelight) diff --git a/VERSION b/VERSION index 1b47653967..d3b74ae5f3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.5-650 +2.5-651 diff --git a/src/analyzer/protocol/ncp/NCP.cc b/src/analyzer/protocol/ncp/NCP.cc index f01c409429..e8672e7ebe 100644 --- a/src/analyzer/protocol/ncp/NCP.cc +++ b/src/analyzer/protocol/ncp/NCP.cc @@ -80,7 +80,7 @@ void NCP_Session::DeliverFrame(const binpac::NCP::ncp_frame* frame) } } -FrameBuffer::FrameBuffer(int header_length) +FrameBuffer::FrameBuffer(size_t header_length) { hdr_len = header_length; msg_buf = 0; diff --git a/src/analyzer/protocol/ncp/NCP.h b/src/analyzer/protocol/ncp/NCP.h index bdf5d8bffe..ff64db9077 100644 --- a/src/analyzer/protocol/ncp/NCP.h +++ b/src/analyzer/protocol/ncp/NCP.h @@ -51,7 +51,7 @@ protected: class FrameBuffer { public: - explicit FrameBuffer(int header_length); + explicit FrameBuffer(size_t header_length); virtual ~FrameBuffer(); // Returns -1 if frame is not ready, 0 if it else, and 1 if @@ -67,7 +67,7 @@ public: protected: virtual void compute_msg_length() = 0; - int hdr_len; + size_t hdr_len; u_char* msg_buf; uint64 msg_len; size_t buf_n; // number of bytes in msg_buf From da593e7aaa8f5b9f59df0be501dc54d7e2fd6b33 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 7 Jun 2018 13:56:02 -0500 Subject: [PATCH 37/56] GH-131: disable krb ticket decryption on non-Linux A test case fails on macOS and FreeBSD fails to build. --- CHANGES | 4 ++++ CMakeLists.txt | 10 ++++++---- VERSION | 2 +- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/CHANGES b/CHANGES index 188dd6439c..cb297c3b31 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.5-652 | 2018-06-07 13:57:23 -0500 + + * GH-131: disable krb ticket decryption on non-Linux (Corelight) + 2.5-651 | 2018-06-07 09:57:29 -0500 * Fix signed/unsigned comparison compiler warning (Corelight) diff --git a/CMakeLists.txt b/CMakeLists.txt index 34fb514cc6..d0ea236330 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -144,10 +144,12 @@ if (LIBGEOIP_FOUND) endif () set(USE_KRB5 false) -find_package(LibKrb5) -if (LibKrb5_FOUND) - set(USE_KRB5 true) - list(APPEND OPTLIBS ${LibKrb5_LIBRARY}) +if ( ${CMAKE_SYSTEM_NAME} MATCHES Linux ) + find_package(LibKrb5) + if (LibKrb5_FOUND) + set(USE_KRB5 true) + list(APPEND OPTLIBS ${LibKrb5_LIBRARY}) + endif () endif () set(HAVE_PERFTOOLS false) diff --git a/VERSION b/VERSION index d3b74ae5f3..8620f5a053 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.5-651 +2.5-652 From c2c35ed17897a5e09b37a4b90169f3ac1409a0ca Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 7 Jun 2018 16:56:14 -0500 Subject: [PATCH 38/56] Updating submodule(s). [nomail] --- aux/broker | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/broker b/aux/broker index 81cf863bb2..9b56fea499 160000 --- a/aux/broker +++ b/aux/broker @@ -1 +1 @@ -Subproject commit 81cf863bb26c39b88f6cf6d1d8439458a1586bee +Subproject commit 9b56fea4999d4e11a5cd2caaafd934759015fab5 From 8bbe84a1b6b0eb33e15031cdcce3cac977edbaa0 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 8 Jun 2018 10:00:42 -0500 Subject: [PATCH 39/56] Updating submodule(s). [nomail] --- aux/broctl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/broctl b/aux/broctl index 64fc253007..99ef926d09 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 64fc25300723d753217b5cf5e8bee2d4500409e1 +Subproject commit 99ef926d09a144a9919dec01f6231692e170120e From b51e6f39ddc641811d4875cda4543d3a60fb5a63 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 8 Jun 2018 10:43:03 -0500 Subject: [PATCH 40/56] Correct conn history field documentation --- CHANGES | 4 ++++ VERSION | 2 +- scripts/base/protocols/conn/main.bro | 9 ++++++--- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/CHANGES b/CHANGES index cb297c3b31..80083ef0c4 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.5-655 | 2018-06-08 10:43:03 -0500 + + * Correct conn history field documentation (Corelight) + 2.5-652 | 2018-06-07 13:57:23 -0500 * GH-131: disable krb ticket decryption on non-Linux (Corelight) diff --git a/VERSION b/VERSION index 8620f5a053..968ce81a87 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.5-652 +2.5-655 diff --git a/scripts/base/protocols/conn/main.bro b/scripts/base/protocols/conn/main.bro index c806a017e0..0e9661dea3 100644 --- a/scripts/base/protocols/conn/main.bro +++ b/scripts/base/protocols/conn/main.bro @@ -95,9 +95,12 @@ export { ## ## If the event comes from the originator, the letter is in ## upper-case; if it comes from the responder, it's in - ## lower-case. Multiple packets of the same type will only be - ## noted once (e.g. we only record one "d" in each direction, - ## regardless of how many data packets were seen.) + ## lower-case. The 'a', 'c', 'd', 'i', 'q', and 't' flags are + ## recorded a maximum of one time in either direction regardless + ## of how many are actually seen. However, 'f', 'h', 'r', or + ## 's' may be recorded multiple times for either direction and + ## only compressed when sharing a sequence number with the + ## last-seen packet of the same flag type. history: string &log &optional; ## Number of packets that the originator sent. ## Only set if :bro:id:`use_conn_size_analyzer` = T. From 6752ffcc8efe2a3dc97586f31f27e39fa0c4d0bd Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 11 Jun 2018 10:39:56 -0500 Subject: [PATCH 41/56] Updating submodule(s). [nomail] --- aux/broker | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/broker b/aux/broker index 9b56fea499..bf8943df55 160000 --- a/aux/broker +++ b/aux/broker @@ -1 +1 @@ -Subproject commit 9b56fea4999d4e11a5cd2caaafd934759015fab5 +Subproject commit bf8943df551efccaaa2b19e838429239260748e8 From c9fe9a943c4d78b18ffbae13c562b93349a5f951 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 12 Jun 2018 13:49:39 -0500 Subject: [PATCH 42/56] Add Broker::max_live_threads and Broker::max_pcap_threads tunables These may be used to change the number of scheduler threads that the underlying CAF library creates. In pcap mode, it's currently hardcoded to the minimal 4 threads due to potentially significant overhead in CAF. --- CHANGES | 5 +++ VERSION | 2 +- scripts/base/frameworks/broker/main.bro | 16 ++++++++++ src/broker/Manager.cc | 42 ++++++++++++++++--------- src/broker/Manager.h | 7 ++++- 5 files changed, 55 insertions(+), 17 deletions(-) diff --git a/CHANGES b/CHANGES index 28bd93ba6b..10d00d919f 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,9 @@ +2.5-660 | 2018-06-12 13:49:39 -0500 + + * Add Broker::max_live_threads and Broker::max_pcap_threads tunables + (Corelight) + 2.5-658 | 2018-06-08 16:41:07 +0000 * Allow BRO_DEFAULT_LISTEN_ADDRESS to control broker listen address. diff --git a/VERSION b/VERSION index 0b13a1e84e..946f73bee2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.5-658 +2.5-660 diff --git a/scripts/base/frameworks/broker/main.bro b/scripts/base/frameworks/broker/main.bro index 5368d5422b..cfd4f06280 100644 --- a/scripts/base/frameworks/broker/main.bro +++ b/scripts/base/frameworks/broker/main.bro @@ -51,6 +51,22 @@ export { ## all peers. const ssl_keyfile = "" &redef; + ## Max number of threads to use for Broker/CAF functionality when + ## operating on a live interface. Using zero will cause this to + ## be automatically determined based on number of available CPUs. + const max_live_threads = 0 &redef; + + ## Max number of threads to use for Broker/CAF functionality when + ## operating on a pcap file. Using zero will cause this to be + ## automaticallu determined based on number of available CPUs. + # TODO: on systems where number of CPUs starts exceeding ~10, + # simply creating a caf::actor_system and not using it incurs + # significant performance overhead. Can CAF be updated to + # be more efficient in the case where the application isn't + # actually making much use of most of those threads instead + # of hardcoding this to the minimal 4 threads? + const max_pcap_threads = 4 &redef; + ## Forward all received messages to subscribing peers. const forward_messages = F &redef; diff --git a/src/broker/Manager.cc b/src/broker/Manager.cc index 3d666fa1d4..f18d245bb0 100644 --- a/src/broker/Manager.cc +++ b/src/broker/Manager.cc @@ -113,21 +113,18 @@ static inline Val* get_option(const char* option) return id->ID_Val(); } -class configuration : public broker::configuration { -public: - configuration(broker::broker_options options) - : broker::configuration(options) - { - openssl_cafile = get_option("Broker::ssl_cafile")->AsString()->CheckString(); - openssl_capath = get_option("Broker::ssl_capath")->AsString()->CheckString(); - openssl_certificate = get_option("Broker::ssl_certificate")->AsString()->CheckString(); - openssl_key = get_option("Broker::ssl_keyfile")->AsString()->CheckString(); - openssl_passphrase = get_option("Broker::ssl_passphrase")->AsString()->CheckString(); - } -}; +Manager::BrokerConfig::BrokerConfig(broker::broker_options options) + : broker::configuration(options) + { + openssl_cafile = get_option("Broker::ssl_cafile")->AsString()->CheckString(); + openssl_capath = get_option("Broker::ssl_capath")->AsString()->CheckString(); + openssl_certificate = get_option("Broker::ssl_certificate")->AsString()->CheckString(); + openssl_key = get_option("Broker::ssl_keyfile")->AsString()->CheckString(); + openssl_passphrase = get_option("Broker::ssl_passphrase")->AsString()->CheckString(); + } -Manager::BrokerState::BrokerState(broker::broker_options options) - : endpoint(configuration(options)), +Manager::BrokerState::BrokerState(BrokerConfig config) + : endpoint(std::move(config)), subscriber(endpoint.make_subscriber({}, SUBSCRIBER_MAX_QSIZE)), status_subscriber(endpoint.make_status_subscriber(true)) { @@ -173,7 +170,22 @@ void Manager::InitPostScript() options.forward = get_option("Broker::forward_messages")->AsBool(); options.use_real_time = ! reading_pcaps; - bstate = std::make_shared(options); + BrokerConfig config{std::move(options)}; + auto max_live_threads = get_option("Broker::max_live_threads")->AsCount(); + auto max_pcap_threads = get_option("Broker::max_pcap_threads")->AsCount(); + + if ( reading_pcaps ) + { + if ( max_pcap_threads ) + config.scheduler_max_threads = max_pcap_threads; + } + else + { + if ( max_live_threads ) + config.scheduler_max_threads = max_live_threads; + } + + bstate = std::make_shared(std::move(config)); } void Manager::Terminate() diff --git a/src/broker/Manager.h b/src/broker/Manager.h index 182203f829..8c5ab09dc6 100644 --- a/src/broker/Manager.h +++ b/src/broker/Manager.h @@ -339,9 +339,14 @@ public: private: + class BrokerConfig : public broker::configuration { + public: + BrokerConfig(broker::broker_options options); + }; + class BrokerState { public: - BrokerState(broker::broker_options options); + BrokerState(BrokerConfig config); broker::endpoint endpoint; broker::subscriber subscriber; broker::status_subscriber status_subscriber; From 8a00a2dbf8414fb381f33859beab321cd4465eda Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 13 Jun 2018 14:48:28 -0500 Subject: [PATCH 43/56] Updating submodule(s). [nomail] --- aux/broker | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/broker b/aux/broker index bf8943df55..ee8c3c19f8 160000 --- a/aux/broker +++ b/aux/broker @@ -1 +1 @@ -Subproject commit bf8943df551efccaaa2b19e838429239260748e8 +Subproject commit ee8c3c19f8e9501623b552df30c8d78c2c66c98d From 197ea03f8f917664d41b63438a38790b4606baf7 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 14 Jun 2018 12:41:21 -0500 Subject: [PATCH 44/56] Minor optimization to bro_broker::Manager::FlushPendingQueries --- src/broker/Manager.cc | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/broker/Manager.cc b/src/broker/Manager.cc index f18d245bb0..3a980d22c2 100644 --- a/src/broker/Manager.cc +++ b/src/broker/Manager.cc @@ -241,8 +241,17 @@ void Manager::FlushPendingQueries() { // possibly an infinite loop if a query can recursively // generate more queries... - Process(); + for ( auto& s : data_stores ) + { + while ( ! s.second->proxy.mailbox().empty() ) + { + auto response = s.second->proxy.receive(); + ProcessStoreResponse(s.second, move(response)); + } + } } + + SetIdle(false); } uint16_t Manager::Listen(const string& addr, uint16_t port) From e578c1c23162889c3dd4963854a465e92ef4e1d5 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 14 Jun 2018 12:51:28 -0500 Subject: [PATCH 45/56] Add Broker::max_threads and Broker::max_sleep tuning options The former replaces the pcap vs. live versions of the same tuning option. If a user does not change these, Bro makes some internal decisions that may help avoid performance problems on systems with high core counts: the number of CAF threads is capped at 8 and the maximum sleep duration for under-utilized threads is increased to 64ms (CAF's default is 10ms). --- CHANGES | 7 +++++ VERSION | 2 +- scripts/base/frameworks/broker/main.bro | 22 +++++--------- src/broker/Manager.cc | 39 +++++++++++++++++++------ 4 files changed, 46 insertions(+), 24 deletions(-) diff --git a/CHANGES b/CHANGES index 10d00d919f..e6692ef2f5 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,11 @@ +2.5-663 | 2018-06-14 12:51:28 -0500 + + * Add Broker::max_threads and Broker::max_sleep tuning options, + remove Broker::max_live_threads and Broker::max_pcap threads (Corelight) + + * Minor optimization to bro_broker::Manager::FlushPendingQueries (Corelight) + 2.5-660 | 2018-06-12 13:49:39 -0500 * Add Broker::max_live_threads and Broker::max_pcap_threads tunables diff --git a/VERSION b/VERSION index 946f73bee2..a38fa6e474 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.5-660 +2.5-663 diff --git a/scripts/base/frameworks/broker/main.bro b/scripts/base/frameworks/broker/main.bro index cfd4f06280..23a701c3ef 100644 --- a/scripts/base/frameworks/broker/main.bro +++ b/scripts/base/frameworks/broker/main.bro @@ -51,21 +51,15 @@ export { ## all peers. const ssl_keyfile = "" &redef; - ## Max number of threads to use for Broker/CAF functionality when - ## operating on a live interface. Using zero will cause this to - ## be automatically determined based on number of available CPUs. - const max_live_threads = 0 &redef; + ## Max number of threads to use for Broker/CAF functionality. + ## Using zero will cause this to be automatically determined + ## based on number of available CPUs. + const max_threads = 0 &redef; - ## Max number of threads to use for Broker/CAF functionality when - ## operating on a pcap file. Using zero will cause this to be - ## automaticallu determined based on number of available CPUs. - # TODO: on systems where number of CPUs starts exceeding ~10, - # simply creating a caf::actor_system and not using it incurs - # significant performance overhead. Can CAF be updated to - # be more efficient in the case where the application isn't - # actually making much use of most of those threads instead - # of hardcoding this to the minimal 4 threads? - const max_pcap_threads = 4 &redef; + ## Max number of microseconds for under-utilized Broker/CAF + ## threads to sleep. Using zero will cause this to be automatically + ## determined or just use CAF's default setting. + const max_sleep = 0 &redef; ## Forward all received messages to subscribing peers. const forward_messages = F &redef; diff --git a/src/broker/Manager.cc b/src/broker/Manager.cc index 3a980d22c2..e78db804a8 100644 --- a/src/broker/Manager.cc +++ b/src/broker/Manager.cc @@ -171,20 +171,41 @@ void Manager::InitPostScript() options.use_real_time = ! reading_pcaps; BrokerConfig config{std::move(options)}; - auto max_live_threads = get_option("Broker::max_live_threads")->AsCount(); - auto max_pcap_threads = get_option("Broker::max_pcap_threads")->AsCount(); + auto max_threads = get_option("Broker::max_threads")->AsCount(); + auto max_sleep = get_option("Broker::max_sleep")->AsCount(); - if ( reading_pcaps ) - { - if ( max_pcap_threads ) - config.scheduler_max_threads = max_pcap_threads; - } + if ( max_threads ) + config.scheduler_max_threads = max_threads; else { - if ( max_live_threads ) - config.scheduler_max_threads = max_live_threads; + // On high-core-count systems, spawning one thread per core + // can lead to significant performance problems even if most + // threads are under-utilized. Related: + // https://github.com/actor-framework/actor-framework/issues/699 + if ( reading_pcaps ) + config.scheduler_max_threads = 2u; + else + { + auto hc = std::thread::hardware_concurrency(); + + if ( hc > 8u ) + hc = 8u; + else if ( hc < 4u) + hc = 4u; + + config.scheduler_max_threads = hc; + } } + if ( max_sleep ) + config.work_stealing_relaxed_sleep_duration_us = max_sleep; + else + // 64ms is just an arbitrary amount derived from testing + // the overhead of a unused CAF actor system on a 32-core system. + // Performance was within 2% of baseline timings (w/o CAF) + // when using this sleep duration. + config.work_stealing_relaxed_sleep_duration_us = 64000; + bstate = std::make_shared(std::move(config)); } From 32bd0bfb6e50d2e741827429e62216d2f9ee4aa6 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 14 Jun 2018 16:32:01 -0500 Subject: [PATCH 46/56] Updating submodule(s). [nomail] --- aux/broker | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/broker b/aux/broker index ee8c3c19f8..d6b28211b8 160000 --- a/aux/broker +++ b/aux/broker @@ -1 +1 @@ -Subproject commit ee8c3c19f8e9501623b552df30c8d78c2c66c98d +Subproject commit d6b28211b8cfb7d2f5ee089d13cb32413d02a5ba From 5ac2cdecd42db419de677b6cfe4334efe96b483a Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 14 Jun 2018 17:34:38 -0500 Subject: [PATCH 47/56] Updating submodule(s). [nomail] --- aux/broker | 2 +- src/3rdparty | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aux/broker b/aux/broker index d6b28211b8..9daf9184b9 160000 --- a/aux/broker +++ b/aux/broker @@ -1 +1 @@ -Subproject commit d6b28211b8cfb7d2f5ee089d13cb32413d02a5ba +Subproject commit 9daf9184b999e67dcd5ed26d7862e73c42d26408 diff --git a/src/3rdparty b/src/3rdparty index 6511cd6e45..7c82dcc67f 160000 --- a/src/3rdparty +++ b/src/3rdparty @@ -1 +1 @@ -Subproject commit 6511cd6e45811af0904947a36e565dcb9eee61dd +Subproject commit 7c82dcc67fb0c53d6e18074dc828875eae1995fa From bf246e59d0fc91d36aafe9f23b9b47470228894f Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 15 Jun 2018 14:57:16 -0500 Subject: [PATCH 48/56] Updating submodule(s). [nomail] --- aux/broker | 2 +- src/3rdparty | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aux/broker b/aux/broker index 9daf9184b9..5ea77922d5 160000 --- a/aux/broker +++ b/aux/broker @@ -1 +1 @@ -Subproject commit 9daf9184b999e67dcd5ed26d7862e73c42d26408 +Subproject commit 5ea77922d500d2a485dabafa9fbe81176e35060e diff --git a/src/3rdparty b/src/3rdparty index 7c82dcc67f..bae2418c89 160000 --- a/src/3rdparty +++ b/src/3rdparty @@ -1 +1 @@ -Subproject commit 7c82dcc67fb0c53d6e18074dc828875eae1995fa +Subproject commit bae2418c89531313aee4dd8809241457b22479e5 From 3c6afc8409524d5321c1ffb556104c117eb040f6 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 15 Jun 2018 15:30:11 -0500 Subject: [PATCH 49/56] Add --disable-broker-tests configure option --- CHANGES | 4 ++++ VERSION | 2 +- aux/broker | 2 +- configure | 4 ++++ 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index e6692ef2f5..8e3547000a 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.5-667 | 2018-06-15 15:30:11 -0500 + + * Add --disable-broker-tests configure option (Corelight) + 2.5-663 | 2018-06-14 12:51:28 -0500 * Add Broker::max_threads and Broker::max_sleep tuning options, diff --git a/VERSION b/VERSION index a38fa6e474..fe9e42523c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.5-663 +2.5-667 diff --git a/aux/broker b/aux/broker index 5ea77922d5..a11f8c6e20 160000 --- a/aux/broker +++ b/aux/broker @@ -1 +1 @@ -Subproject commit 5ea77922d500d2a485dabafa9fbe81176e35060e +Subproject commit a11f8c6e20264abe6147bc2b1be7ca3e7dabad05 diff --git a/configure b/configure index ed4cbf3d6e..fdbca263c6 100755 --- a/configure +++ b/configure @@ -55,6 +55,7 @@ Usage: $0 [OPTION]... [VAR=VALUE]... --disable-auxtools don't build or install auxiliary tools --disable-perftools don't try to build with Google Perftools --disable-python don't try to build python bindings for broker + --disable-broker-tests don'e try to build Broker unit tests Required Packages in Non-Standard Locations: --with-openssl=PATH path to OpenSSL install root @@ -227,6 +228,9 @@ while [ $# -ne 0 ]; do --disable-python) append_cache_entry DISABLE_PYTHON_BINDINGS BOOL true ;; + --disable-broker-tests) + append_cache_entry BROKER_DISABLE_TESTS BOOL true + ;; --with-openssl=*) append_cache_entry OPENSSL_ROOT_DIR PATH $optarg ;; From c11039cb73e5228f79a6430e2ab5c29cb27c1eec Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 15 Jun 2018 17:14:33 -0500 Subject: [PATCH 50/56] Make old comm. system usages an error unless old_comm_usage_is_ok is set --- CHANGES | 5 ++ VERSION | 2 +- doc/frameworks/broker.rst | 7 ++ scripts/base/init-bare.bro | 7 ++ src/Net.h | 2 + src/main.cc | 82 +++++++++++++++++++ src/scan.l | 1 + .../btest/Baseline/core.old_comm_usage/out | 2 + testing/btest/core/old_comm_usage.bro | 7 ++ 9 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 testing/btest/Baseline/core.old_comm_usage/out create mode 100644 testing/btest/core/old_comm_usage.bro diff --git a/CHANGES b/CHANGES index 8e3547000a..e1dbc6bfcf 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,9 @@ +2.5-668 | 2018-06-15 17:14:33 -0500 + + * Make old comm. system usages an error unless old_comm_usage_is_ok is set + (Corelight) + 2.5-667 | 2018-06-15 15:30:11 -0500 * Add --disable-broker-tests configure option (Corelight) diff --git a/VERSION b/VERSION index fe9e42523c..483a2c482b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.5-667 +2.5-668 diff --git a/doc/frameworks/broker.rst b/doc/frameworks/broker.rst index 807ce9a07a..6943a0a698 100644 --- a/doc/frameworks/broker.rst +++ b/doc/frameworks/broker.rst @@ -50,6 +50,13 @@ General Porting Tips - The ``&synchronized`` and ``&persistent`` attributes are deprecated, consider using `Data Stores`_ instead. +- Usages of the old communications system features are all deprecated, + however, they also do not work in the default Bro configuration unless + you manually take action to set up the old communication system. + To aid in porting, such usages will default to raising a fatal error + unless you explicitly acknowledge that such usages of the old system + are ok. Set the :bro:see:`old_comm_usage_is_ok`` flag in this case. + - Instead of using e.g. ``Cluster::manager2worker_events`` (and all permutations for every node type), what you'd now use is either :bro:see:`Broker::publish` or :bro:see:`Broker::auto_publish` with diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index cd9302ce25..c502607cbd 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -4834,3 +4834,10 @@ const global_hash_seed: string = "" &redef; ## files. The larger the value, the more confidence in UID uniqueness. ## The maximum is currently 128 bits. const bits_per_uid: count = 96 &redef; + +## Whether usage of the old communication system is considered an error or +## not. The default Bro configuration no longer works with the non-Broker +## communication system unless you have manually taken action to initialize +## and set up the old comm. system. Deprecation warnings are still emitted +## when setting this flag, but they will not result in a fatal error. +const old_comm_usage_is_ok: bool = F &redef; diff --git a/src/Net.h b/src/Net.h index caea61c436..bdc84ec74f 100644 --- a/src/Net.h +++ b/src/Net.h @@ -83,6 +83,8 @@ extern iosource::PktDumper* pkt_dumper; // where to save packets extern char* writefile; +extern int old_comm_usage_count; + // Script file we have already scanned (or are in the process of scanning). // They are identified by inode number. struct ScannedFile { diff --git a/src/main.cc b/src/main.cc index 2a61c753b8..2e9a89ddd1 100644 --- a/src/main.cc +++ b/src/main.cc @@ -44,6 +44,7 @@ extern "C" { #include "EventRegistry.h" #include "Stats.h" #include "Brofiler.h" +#include "Traverse.h" #include "threading/Manager.h" #include "input/Manager.h" @@ -114,6 +115,7 @@ char* command_line_policy = 0; vector params; set requested_plugins; char* proc_status_file = 0; +int old_comm_usage_count = 0; OpaqueType* md5_type = 0; OpaqueType* sha1_type = 0; @@ -424,6 +426,70 @@ static void bro_new_handler() out_of_memory("new"); } +static auto old_comm_ids = std::set{ + "connect", + "disconnect", + "request_remote_events", + "request_remote_sync", + "request_remote_logs", + "set_accept_state", + "set_compression_level", + "listen", + "send_id", + "terminate_communication", + "complete_handshake", + "send_ping", + "send_current_packet", + "get_event_peer", + "send_capture_filter", + "suspend_state_updates", + "resume_state_updates", +}; + +static bool is_old_comm_usage(const ID* id) + { + auto name = id->Name(); + + if ( old_comm_ids.find(name) == old_comm_ids.end() ) + return false; + + return true; + } + +class OldCommUsageTraversalCallback : public TraversalCallback { +public: + virtual TraversalCode PreExpr(const Expr* expr) override + { + switch ( expr->Tag() ) { + case EXPR_CALL: + { + const CallExpr* call = static_cast(expr); + auto func = call->Func(); + + if ( func->Tag() == EXPR_NAME ) + { + const NameExpr* ne = static_cast(func); + auto id = ne->Id(); + + if ( is_old_comm_usage(id) ) + ++old_comm_usage_count; + } + } + break; + default: + break; + } + + return TC_CONTINUE; + } +}; + +static void find_old_comm_usages() + { + OldCommUsageTraversalCallback cb; + traverse_all(&cb); + } + int main(int argc, char** argv) { std::set_new_handler(bro_new_handler); @@ -854,6 +920,22 @@ int main(int argc, char** argv) yyparse(); is_parsing = false; + find_old_comm_usages(); + + if ( old_comm_usage_count ) + { + auto old_comm_ack_id = global_scope()->Lookup("old_comm_usage_is_ok"); + + if ( ! old_comm_ack_id->ID_Val()->AsBool() ) + reporter->FatalError("Detected old, deprecated communication " + "system usages that will not work unless " + "you explicitly take action to initizialize " + "and set up the old comm. system. " + "Set the 'old_comm_usage_is_ok' flag " + "to bypass this error if you've taken such " + "actions."); + } + RecordVal::ResizeParseTimeRecords(); init_general_global_var(); diff --git a/src/scan.l b/src/scan.l index 27490c13ad..ed307a79da 100644 --- a/src/scan.l +++ b/src/scan.l @@ -310,6 +310,7 @@ when return TOK_WHEN; } &synchronized { + ++old_comm_usage_count; deprecated_attr(yytext); return TOK_ATTR_SYNCHRONIZED; } diff --git a/testing/btest/Baseline/core.old_comm_usage/out b/testing/btest/Baseline/core.old_comm_usage/out new file mode 100644 index 0000000000..28585d78ba --- /dev/null +++ b/testing/btest/Baseline/core.old_comm_usage/out @@ -0,0 +1,2 @@ +warning in /Users/jon/projects/bro/bro/testing/btest/.tmp/core.old_comm_usage/old_comm_usage.bro, line 6: deprecated (terminate_communication) +fatal error: Detected old, deprecated communication system usages that will not work unless you explicitly take action to initizialize and set up the old comm. system. Set the 'old_comm_usage_is_ok' flag to bypass this error if you've taken such actions. diff --git a/testing/btest/core/old_comm_usage.bro b/testing/btest/core/old_comm_usage.bro new file mode 100644 index 0000000000..0e9ae2f1f6 --- /dev/null +++ b/testing/btest/core/old_comm_usage.bro @@ -0,0 +1,7 @@ +# @TEST-EXEC-FAIL: bro -b %INPUT >out 2>&1 +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out + +event bro_init() + { + terminate_communication(); + } From 344382ee7beda7c610af297a1a5aa58e8af4f9a9 Mon Sep 17 00:00:00 2001 From: Vern Paxson Date: Tue, 19 Jun 2018 11:50:38 -0700 Subject: [PATCH 51/56] documentation clarification for "p1 | p2" --- doc/script-reference/types.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/script-reference/types.rst b/doc/script-reference/types.rst index fe77ece90c..b446e972dd 100644 --- a/doc/script-reference/types.rst +++ b/doc/script-reference/types.rst @@ -237,12 +237,13 @@ Here is a more detailed description of each type: is false since "oob" does not appear at the start of "foobar". The ``!in`` operator would yield the negation of ``in``. - Finally, you can create a disjunction (either-or) of two patterns + Finally, you can create a disjunction (either-or) of two literal patterns using the ``|`` operator. For example:: /foo/ | /bar/ in "foobar" - yields true, like in the similar example above. + yields true, like in the similar example above. (This does not presently + work for variables whose values are patterns, however.) .. bro:type:: port From 7b3bad635ba46a5583acae6f5b8ef440cc14694a Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Tue, 19 Jun 2018 16:26:51 -0500 Subject: [PATCH 52/56] Disable broxygen when running unit tests Disable broxygen when running unit tests (except for the tests that use broxygen). On my dual-core MacBook Pro, this change results in the unit tests taking about 13% less time to finish running. --- testing/btest/btest.cfg | 1 + testing/btest/doc/broxygen/all_scripts.test | 2 +- testing/btest/doc/broxygen/command_line.bro | 2 +- testing/btest/doc/broxygen/comment_retrieval_bifs.bro | 2 +- testing/btest/doc/broxygen/enums.bro | 2 +- testing/btest/doc/broxygen/example.bro | 2 +- testing/btest/doc/broxygen/func-params.bro | 2 +- testing/btest/doc/broxygen/identifier.bro | 2 +- testing/btest/doc/broxygen/package.bro | 2 +- testing/btest/doc/broxygen/package_index.bro | 2 +- testing/btest/doc/broxygen/records.bro | 2 +- testing/btest/doc/broxygen/script_index.bro | 2 +- testing/btest/doc/broxygen/script_summary.bro | 2 +- testing/btest/doc/broxygen/type-aliases.bro | 2 +- testing/btest/doc/broxygen/vectors.bro | 2 +- 15 files changed, 15 insertions(+), 14 deletions(-) diff --git a/testing/btest/btest.cfg b/testing/btest/btest.cfg index 81ee5bd5e5..6671d70b64 100644 --- a/testing/btest/btest.cfg +++ b/testing/btest/btest.cfg @@ -26,3 +26,4 @@ BRO_PROFILER_FILE=%(testbase)s/.tmp/script-coverage/XXXXXX BTEST_RST_FILTER=$SCRIPTS/rst-filter BRO_DNS_FAKE=1 BRO_DEFAULT_LISTEN_ADDRESS=127.0.0.1 +BRO_DISABLE_BROXYGEN=1 diff --git a/testing/btest/doc/broxygen/all_scripts.test b/testing/btest/doc/broxygen/all_scripts.test index c0cb07b750..dc009044da 100644 --- a/testing/btest/doc/broxygen/all_scripts.test +++ b/testing/btest/doc/broxygen/all_scripts.test @@ -5,7 +5,7 @@ # listen.bro in order to document it. # @TEST-SERIALIZE: comm -# @TEST-EXEC: bro -X broxygen.config broxygen DumpEvents::include=/NOTHING_MATCHES/ +# @TEST-EXEC: unset BRO_DISABLE_BROXYGEN; bro -X broxygen.config broxygen DumpEvents::include=/NOTHING_MATCHES/ # @TEST-EXEC: btest-diff .stdout # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderr diff --git a/testing/btest/doc/broxygen/command_line.bro b/testing/btest/doc/broxygen/command_line.bro index 95558f7461..d009667b7e 100644 --- a/testing/btest/doc/broxygen/command_line.bro +++ b/testing/btest/doc/broxygen/command_line.bro @@ -1,7 +1,7 @@ # Shouldn't emit any warnings about not being able to document something # that's supplied via command line script. -# @TEST-EXEC: bro %INPUT -e 'redef myvar=10; print myvar' >output 2>&1 +# @TEST-EXEC: unset BRO_DISABLE_BROXYGEN; bro %INPUT -e 'redef myvar=10; print myvar' >output 2>&1 # @TEST-EXEC: btest-diff output const myvar = 5 &redef; diff --git a/testing/btest/doc/broxygen/comment_retrieval_bifs.bro b/testing/btest/doc/broxygen/comment_retrieval_bifs.bro index 77a6058d71..f3c1be6b14 100644 --- a/testing/btest/doc/broxygen/comment_retrieval_bifs.bro +++ b/testing/btest/doc/broxygen/comment_retrieval_bifs.bro @@ -1,4 +1,4 @@ -# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: unset BRO_DISABLE_BROXYGEN; bro -b %INPUT >out # @TEST-EXEC: btest-diff out ##! This is a test script. diff --git a/testing/btest/doc/broxygen/enums.bro b/testing/btest/doc/broxygen/enums.bro index e8b4c741c2..8fbdb11ab6 100644 --- a/testing/btest/doc/broxygen/enums.bro +++ b/testing/btest/doc/broxygen/enums.bro @@ -1,4 +1,4 @@ -# @TEST-EXEC: bro -b -X broxygen.config %INPUT +# @TEST-EXEC: unset BRO_DISABLE_BROXYGEN; bro -b -X broxygen.config %INPUT # @TEST-EXEC: btest-diff autogen-reST-enums.rst @TEST-START-FILE broxygen.config diff --git a/testing/btest/doc/broxygen/example.bro b/testing/btest/doc/broxygen/example.bro index e7212f3c5f..22a6fc7418 100644 --- a/testing/btest/doc/broxygen/example.bro +++ b/testing/btest/doc/broxygen/example.bro @@ -1,4 +1,4 @@ -# @TEST-EXEC: bro -X broxygen.config %INPUT +# @TEST-EXEC: unset BRO_DISABLE_BROXYGEN; bro -X broxygen.config %INPUT # @TEST-EXEC: btest-diff example.rst @TEST-START-FILE broxygen.config diff --git a/testing/btest/doc/broxygen/func-params.bro b/testing/btest/doc/broxygen/func-params.bro index 42d1308151..e53ca475f1 100644 --- a/testing/btest/doc/broxygen/func-params.bro +++ b/testing/btest/doc/broxygen/func-params.bro @@ -1,4 +1,4 @@ -# @TEST-EXEC: bro -b -X broxygen.config %INPUT +# @TEST-EXEC: unset BRO_DISABLE_BROXYGEN; bro -b -X broxygen.config %INPUT # @TEST-EXEC: btest-diff autogen-reST-func-params.rst @TEST-START-FILE broxygen.config diff --git a/testing/btest/doc/broxygen/identifier.bro b/testing/btest/doc/broxygen/identifier.bro index db5c2528ee..9225ab8db0 100644 --- a/testing/btest/doc/broxygen/identifier.bro +++ b/testing/btest/doc/broxygen/identifier.bro @@ -1,5 +1,5 @@ # @TEST-SERIALIZE: comm -# @TEST-EXEC: bro -b -X broxygen.config %INPUT +# @TEST-EXEC: unset BRO_DISABLE_BROXYGEN; bro -b -X broxygen.config %INPUT # @TEST-EXEC: btest-diff test.rst @TEST-START-FILE broxygen.config diff --git a/testing/btest/doc/broxygen/package.bro b/testing/btest/doc/broxygen/package.bro index fd75a1ce21..0d8d790361 100644 --- a/testing/btest/doc/broxygen/package.bro +++ b/testing/btest/doc/broxygen/package.bro @@ -1,5 +1,5 @@ # @TEST-SERIALIZE: comm -# @TEST-EXEC: bro -b -X broxygen.config %INPUT +# @TEST-EXEC: unset BRO_DISABLE_BROXYGEN; bro -b -X broxygen.config %INPUT # @TEST-EXEC: btest-diff test.rst @TEST-START-FILE broxygen.config diff --git a/testing/btest/doc/broxygen/package_index.bro b/testing/btest/doc/broxygen/package_index.bro index ef6cc4ab29..4b98290f20 100644 --- a/testing/btest/doc/broxygen/package_index.bro +++ b/testing/btest/doc/broxygen/package_index.bro @@ -1,5 +1,5 @@ # @TEST-SERIALIZE: comm -# @TEST-EXEC: bro -b -X broxygen.config %INPUT +# @TEST-EXEC: unset BRO_DISABLE_BROXYGEN; bro -b -X broxygen.config %INPUT # @TEST-EXEC: btest-diff test.rst @TEST-START-FILE broxygen.config diff --git a/testing/btest/doc/broxygen/records.bro b/testing/btest/doc/broxygen/records.bro index 0cc7d27500..fbaa957a9f 100644 --- a/testing/btest/doc/broxygen/records.bro +++ b/testing/btest/doc/broxygen/records.bro @@ -1,4 +1,4 @@ -# @TEST-EXEC: bro -b -X broxygen.config %INPUT +# @TEST-EXEC: unset BRO_DISABLE_BROXYGEN; bro -b -X broxygen.config %INPUT # @TEST-EXEC: btest-diff autogen-reST-records.rst @TEST-START-FILE broxygen.config diff --git a/testing/btest/doc/broxygen/script_index.bro b/testing/btest/doc/broxygen/script_index.bro index 86e1909863..c987c005be 100644 --- a/testing/btest/doc/broxygen/script_index.bro +++ b/testing/btest/doc/broxygen/script_index.bro @@ -1,5 +1,5 @@ # @TEST-SERIALIZE: comm -# @TEST-EXEC: bro -b -X broxygen.config %INPUT +# @TEST-EXEC: unset BRO_DISABLE_BROXYGEN; bro -b -X broxygen.config %INPUT # @TEST-EXEC: btest-diff test.rst @TEST-START-FILE broxygen.config diff --git a/testing/btest/doc/broxygen/script_summary.bro b/testing/btest/doc/broxygen/script_summary.bro index a7aafc65a0..7fc89c3735 100644 --- a/testing/btest/doc/broxygen/script_summary.bro +++ b/testing/btest/doc/broxygen/script_summary.bro @@ -1,5 +1,5 @@ # @TEST-SERIALIZE: comm -# @TEST-EXEC: bro -b -X broxygen.config %INPUT +# @TEST-EXEC: unset BRO_DISABLE_BROXYGEN; bro -b -X broxygen.config %INPUT # @TEST-EXEC: btest-diff test.rst @TEST-START-FILE broxygen.config diff --git a/testing/btest/doc/broxygen/type-aliases.bro b/testing/btest/doc/broxygen/type-aliases.bro index 28c2cc5568..0971327c2b 100644 --- a/testing/btest/doc/broxygen/type-aliases.bro +++ b/testing/btest/doc/broxygen/type-aliases.bro @@ -1,4 +1,4 @@ -# @TEST-EXEC: bro -b -X broxygen.config %INPUT +# @TEST-EXEC: unset BRO_DISABLE_BROXYGEN; bro -b -X broxygen.config %INPUT # @TEST-EXEC: btest-diff autogen-reST-type-aliases.rst @TEST-START-FILE broxygen.config diff --git a/testing/btest/doc/broxygen/vectors.bro b/testing/btest/doc/broxygen/vectors.bro index 62fb31d436..7c18225357 100644 --- a/testing/btest/doc/broxygen/vectors.bro +++ b/testing/btest/doc/broxygen/vectors.bro @@ -1,4 +1,4 @@ -# @TEST-EXEC: bro -b -X broxygen.config %INPUT +# @TEST-EXEC: unset BRO_DISABLE_BROXYGEN; bro -b -X broxygen.config %INPUT # @TEST-EXEC: btest-diff autogen-reST-vectors.rst @TEST-START-FILE broxygen.config From 66ee3764117e4d01c1ed3e0ff05ca60161ea9a69 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 21 Jun 2018 11:48:10 -0500 Subject: [PATCH 53/56] BIT-1938: fix crash in Broker manager shutdown --- src/broker/Manager.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/broker/Manager.cc b/src/broker/Manager.cc index e78db804a8..20bba4426d 100644 --- a/src/broker/Manager.cc +++ b/src/broker/Manager.cc @@ -226,7 +226,9 @@ void Manager::Terminate() FlushLogBuffers(); for ( auto& p : bstate->endpoint.peers() ) - bstate->endpoint.unpeer(p.peer.network->address, p.peer.network->port); + if ( p.peer.network ) + bstate->endpoint.unpeer(p.peer.network->address, + p.peer.network->port); bstate->endpoint.shutdown(); } From 8ee92c70da43fa62921a51d4818dce5ea01c96d4 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 21 Jun 2018 11:55:39 -0500 Subject: [PATCH 54/56] Add ability for BroControl to skip cluster setup It does this by setting the BROCTL_CHECK_CONFIG env. variable. Related to BIT-1938. --- CHANGES | 8 ++++++++ VERSION | 2 +- aux/broctl | 2 +- scripts/base/frameworks/cluster/setup-connections.bro | 3 +++ 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index e1dbc6bfcf..69976ba603 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,12 @@ +2.5-671 | 2018-06-21 11:55:39 -0500 + + * Add ability for BroControl to skip cluster setup (Corelight) + + * BIT-1938: fix crash in Broker manager shutdown (Corelight) + + * Disable broxygen when running unit tests (Daniel Thayer) + 2.5-668 | 2018-06-15 17:14:33 -0500 * Make old comm. system usages an error unless old_comm_usage_is_ok is set diff --git a/VERSION b/VERSION index 483a2c482b..fd8c8172cf 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.5-668 +2.5-671 diff --git a/aux/broctl b/aux/broctl index 99ef926d09..c5dd2ba83d 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 99ef926d09a144a9919dec01f6231692e170120e +Subproject commit c5dd2ba83dda185d2008731a5cd25b2b8131ac78 diff --git a/scripts/base/frameworks/cluster/setup-connections.bro b/scripts/base/frameworks/cluster/setup-connections.bro index a06d66bc0c..63ddbdd8b0 100644 --- a/scripts/base/frameworks/cluster/setup-connections.bro +++ b/scripts/base/frameworks/cluster/setup-connections.bro @@ -44,6 +44,9 @@ function connect_peers_with_type(node_type: NodeType) event bro_init() &priority=-10 { + if ( getenv("BROCTL_CHECK_CONFIG") != "" ) + return; + local self = nodes[node]; for ( i in registered_pools ) From 0ab550d3b128418a561df9c690497a1ff860940a Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 21 Jun 2018 14:36:42 -0500 Subject: [PATCH 55/56] Updating submodule(s). [nomail] --- aux/broker | 2 +- src/3rdparty | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aux/broker b/aux/broker index a11f8c6e20..08f41ccc24 160000 --- a/aux/broker +++ b/aux/broker @@ -1 +1 @@ -Subproject commit a11f8c6e20264abe6147bc2b1be7ca3e7dabad05 +Subproject commit 08f41ccc2497f4c6567da0b95488593c39a12a01 diff --git a/src/3rdparty b/src/3rdparty index bae2418c89..c78abc8454 160000 --- a/src/3rdparty +++ b/src/3rdparty @@ -1 +1 @@ -Subproject commit bae2418c89531313aee4dd8809241457b22479e5 +Subproject commit c78abc8454932019f030045340348560a8ac9b23 From 247095479526b61b9b11924027b5a37af199e09d Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Thu, 21 Jun 2018 14:21:36 -0700 Subject: [PATCH 56/56] Fix for ancient reference-counting bug in NFA.cc This patch fixes a *19 year old* reference-counting bug (which could lead to a use-after-free, not merely a leak) in the regular expression code. Patch by Vern Paxson. --- src/NFA.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/NFA.cc b/src/NFA.cc index 43ec3d2a90..8fb78a7131 100644 --- a/src/NFA.cc +++ b/src/NFA.cc @@ -55,7 +55,10 @@ void NFA_State::AddXtionsTo(NFA_state_list* ns) NFA_State* NFA_State::DeepCopy() { if ( mark ) + { + Ref(mark); return mark; + } NFA_State* copy = ccl ? new NFA_State(ccl) : new NFA_State(sym, 0); SetMark(copy);